Reorganize upb file structure This change moves almost everything in the `upb/` directory up one level, so that for example `upb/upb/generated_code_support.h` becomes just `upb/generated_code_support.h`. The only exceptions I made to this were that I left `upb/cmake` and `upb/BUILD` where they are, mostly because that avoids conflict with other files and the current locations seem reasonable for now. The `python/` directory is a little bit of a challenge because we had to merge the existing directory there with `upb/python/`. I made `upb/python/BUILD` into the BUILD file for the merged directory, and it effectively loads the contents of the other BUILD file via `python/build_targets.bzl`, but I plan to clean this up soon. PiperOrigin-RevId: 568651768
diff --git a/python/BUILD b/python/BUILD new file mode 100644 index 0000000..c1e044c --- /dev/null +++ b/python/BUILD
@@ -0,0 +1,233 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("//python:py_extension.bzl", "py_extension") +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") +load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") + +# begin:github_only +load("@rules_pkg//:mappings.bzl", "pkg_files") +load("//python:build_targets.bzl", "build_targets") +build_targets(name = "python") +# end:github_only + +licenses(["notice"]) + +package( + # begin:google_only +# default_applicable_licenses = ["//upb:license"], + # end:google_only + default_visibility = ["//python/dist:__pkg__"], +) + +LIMITED_API_FLAG_SELECT = { + ":limited_api_3.7": ["-DPy_LIMITED_API=0x03070000"], + ":limited_api_3.10": ["-DPy_LIMITED_API=0x030a0000"], + "//conditions:default": [], +} + +bool_flag( + name = "limited_api", + build_setting_default = True, +) + +string_flag( + name = "python_version", + build_setting_default = "system", + values = [ + "system", + "37", + "38", + "39", + "310", + ], +) + +config_setting( + name = "limited_api_3.7", + flag_values = { + ":limited_api": "True", + ":python_version": "37", + }, +) + +config_setting( + name = "full_api_3.7_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "37", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.7_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "37", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.7", + match_any = [ + ":full_api_3.7_win32", + ":full_api_3.7_win64", + ], +) + +config_setting( + name = "full_api_3.8_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "38", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.8_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "38", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.8", + match_any = [ + ":full_api_3.8_win32", + ":full_api_3.8_win64", + ], +) + +config_setting( + name = "full_api_3.9_win32", + flag_values = { + ":limited_api": "False", + ":python_version": "39", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "full_api_3.9_win64", + flag_values = { + ":limited_api": "False", + ":python_version": "39", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "full_api_3.9", + match_any = [ + "full_api_3.9_win32", + ":full_api_3.9_win64", + ], +) + +config_setting( + name = "limited_api_3.10_win32", + flag_values = { + ":limited_api": "True", + ":python_version": "310", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "limited_api_3.10_win64", + flag_values = { + ":limited_api": "True", + ":python_version": "310", + }, + values = {"cpu": "win64"}, +) + +selects.config_setting_group( + name = "limited_api_3.10", + match_any = [ + ":limited_api_3.10_win32", + ":limited_api_3.10_win64", + ], +) + +# begin:github_only +_message_target_compatible_with = { + "@platforms//os:windows": ["@platforms//:incompatible"], + "@system_python//:none": ["@platforms//:incompatible"], + "@system_python//:unsupported": ["@platforms//:incompatible"], + "//conditions:default": [], +} + +# end:github_only +# begin:google_only +# _message_target_compatible_with = { +# "@platforms//os:windows": ["@platforms//:incompatible"], +# "//conditions:default": [], +# } +# end:google_only + +filegroup( + name = "message_srcs", + srcs = [ + "convert.c", + "convert.h", + "descriptor.c", + "descriptor.h", + "descriptor_containers.c", + "descriptor_containers.h", + "descriptor_pool.c", + "descriptor_pool.h", + "extension_dict.c", + "extension_dict.h", + "map.c", + "map.h", + "message.c", + "message.h", + "protobuf.c", + "protobuf.h", + "python_api.h", + "repeated.c", + "repeated.h", + "unknown_fields.c", + "unknown_fields.h", + ], + # begin:google_only +# compatible_with = ["//buildenv/target:non_prod"], + # end:google_only +) + +py_extension( + name = "_message", + srcs = [":message_srcs"], + copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [ + # The Python API requires patterns that are ISO C incompatible, like + # casts between function pointers and object pointers. + "-Wno-pedantic", + ], + target_compatible_with = select(_message_target_compatible_with), + deps = [ + "//upb:collections", + "//upb:descriptor_upb_proto_reflection", + "//upb:eps_copy_input_stream", + "//upb:hash", + "//upb:message_copy", + "//upb:port", + "//upb:reflection", + "//upb:text", + "//upb:wire_reader", + "//upb:wire_types", + "//upb/util:compare", + "//upb/util:def_to_proto", + "//upb/util:required_fields", + ], +)
diff --git a/python/BUILD.bazel b/python/BUILD.bazel deleted file mode 100644 index b42d0b2..0000000 --- a/python/BUILD.bazel +++ /dev/null
@@ -1,6 +0,0 @@ -load("//python:build_targets.bzl", "build_targets") - -# The build targets for this package have been temporarily moved into a Bazel -# macro to facilitate merging upb's Python support into this directory. Once -# that merge is complete, we will move the build targets back here. -build_targets(name = "python")
diff --git a/python/README.md b/python/README.md index baa58c2..ab39350 100644 --- a/python/README.md +++ b/python/README.md
@@ -21,8 +21,8 @@ can use the following Bazel commands: ``` -$ bazel build //upb/python/dist:source_wheel -$ bazel build //upb/python/dist:binary_wheel +$ bazel build //python/dist:source_wheel +$ bazel build //python/dist:binary_wheel ``` The binary wheel will build against whatever version of Python is installed on @@ -43,13 +43,12 @@ following values: 1. **upb**: Built on the - [upb C library](https://github.com/protocolbuffers/upb), this is a new - extension module + [upb C library](https://github.com/protocolbuffers/protobuf/tree/main/upb), + this is a new extension module [released in 4.21.0](https://protobuf.dev/news/2022-05-06/). It offers better performance than any of the previous backends, and it is now the default. It is distributed in our PyPI packages, and requires no special - installation. The code for this module lives in - [upb/python](https://github.com/protocolbuffers/protobuf/tree/main/upb/python). + installation. The code for this module lives in this directory. 1. **cpp**: This extension module wraps the C++ protobuf library. It is deprecated and is no longer released in our PyPI packages, however it is still used in some legacy cases where apps want to perform zero-copy message
diff --git a/python/build_targets.bzl b/python/build_targets.bzl index 546ef01..026292c 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl
@@ -444,7 +444,7 @@ "tox.ini", ], strip_prefix = "", - visibility = ["//upb:__subpackages__"], + visibility = ["//python/dist:__pkg__"], ) pkg_files( @@ -456,9 +456,10 @@ "google/protobuf/pyext/*.cc", "google/protobuf/pyext/*.h", ]) + [ - "BUILD.bazel", + "BUILD", "MANIFEST.in", "README.md", + "build_targets.bzl", "google/protobuf/proto_api.h", "google/protobuf/pyext/README", "google/protobuf/python_protobuf.h",
diff --git a/python/convert.c b/python/convert.c new file mode 100644 index 0000000..98d9b75 --- /dev/null +++ b/python/convert.c
@@ -0,0 +1,446 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/convert.h" + +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/collections/map.h" +#include "upb/reflection/message.h" +#include "upb/util/compare.h" + +// Must be last. +#include "upb/port/def.inc" + +PyObject* PyUpb_UpbToPy(upb_MessageValue val, const upb_FieldDef* f, + PyObject* arena) { + switch (upb_FieldDef_CType(f)) { + case kUpb_CType_Enum: + case kUpb_CType_Int32: + return PyLong_FromLong(val.int32_val); + case kUpb_CType_Int64: + return PyLong_FromLongLong(val.int64_val); + case kUpb_CType_UInt32: + return PyLong_FromSize_t(val.uint32_val); + case kUpb_CType_UInt64: + return PyLong_FromUnsignedLongLong(val.uint64_val); + case kUpb_CType_Float: + return PyFloat_FromDouble(val.float_val); + case kUpb_CType_Double: + return PyFloat_FromDouble(val.double_val); + case kUpb_CType_Bool: + return PyBool_FromLong(val.bool_val); + case kUpb_CType_Bytes: + return PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size); + case kUpb_CType_String: { + PyObject* ret = + PyUnicode_DecodeUTF8(val.str_val.data, val.str_val.size, NULL); + // If the string can't be decoded in UTF-8, just return a bytes object + // that contains the raw bytes. This can't happen if the value was + // assigned using the members of the Python message object, but can happen + // if the values were parsed from the wire (binary). + if (ret == NULL) { + PyErr_Clear(); + ret = PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size); + } + return ret; + } + case kUpb_CType_Message: + return PyUpb_Message_Get((upb_Message*)val.msg_val, + upb_FieldDef_MessageSubDef(f), arena); + default: + PyErr_Format(PyExc_SystemError, + "Getting a value from a field of unknown type %d", + upb_FieldDef_CType(f)); + return NULL; + } +} + +static bool PyUpb_GetInt64(PyObject* obj, int64_t* val) { + // We require that the value is either an integer or has an __index__ + // conversion. + obj = PyNumber_Index(obj); + if (!obj) return false; + // If the value is already a Python long, PyLong_AsLongLong() retrieves it. + // Otherwise is converts to integer using __int__. + *val = PyLong_AsLongLong(obj); + bool ok = true; + if (PyErr_Occurred()) { + assert(PyErr_ExceptionMatches(PyExc_OverflowError)); + PyErr_Clear(); + PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj); + ok = false; + } + Py_DECREF(obj); + return ok; +} + +static bool PyUpb_GetUint64(PyObject* obj, uint64_t* val) { + // We require that the value is either an integer or has an __index__ + // conversion. + obj = PyNumber_Index(obj); + if (!obj) return false; + *val = PyLong_AsUnsignedLongLong(obj); + bool ok = true; + if (PyErr_Occurred()) { + assert(PyErr_ExceptionMatches(PyExc_OverflowError)); + PyErr_Clear(); + PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj); + ok = false; + } + Py_DECREF(obj); + return ok; +} + +static bool PyUpb_GetInt32(PyObject* obj, int32_t* val) { + int64_t i64; + if (!PyUpb_GetInt64(obj, &i64)) return false; + if (i64 < INT32_MIN || i64 > INT32_MAX) { + PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj); + return false; + } + *val = i64; + return true; +} + +static bool PyUpb_GetUint32(PyObject* obj, uint32_t* val) { + uint64_t u64; + if (!PyUpb_GetUint64(obj, &u64)) return false; + if (u64 > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj); + return false; + } + *val = u64; + return true; +} + +// If `arena` is specified, copies the string data into the given arena. +// Otherwise aliases the given data. +static upb_MessageValue PyUpb_MaybeCopyString(const char* ptr, size_t size, + upb_Arena* arena) { + upb_MessageValue ret; + ret.str_val.size = size; + if (arena) { + char* buf = upb_Arena_Malloc(arena, size); + memcpy(buf, ptr, size); + ret.str_val.data = buf; + } else { + ret.str_val.data = ptr; + } + return ret; +} + +const char* upb_FieldDef_TypeString(const upb_FieldDef* f) { + switch (upb_FieldDef_CType(f)) { + case kUpb_CType_Double: + return "double"; + case kUpb_CType_Float: + return "float"; + case kUpb_CType_Int64: + return "int64"; + case kUpb_CType_Int32: + return "int32"; + case kUpb_CType_UInt64: + return "uint64"; + case kUpb_CType_UInt32: + return "uint32"; + case kUpb_CType_Enum: + return "enum"; + case kUpb_CType_Bool: + return "bool"; + case kUpb_CType_String: + return "string"; + case kUpb_CType_Bytes: + return "bytes"; + case kUpb_CType_Message: + return "message"; + } + UPB_UNREACHABLE(); +} + +static bool PyUpb_PyToUpbEnum(PyObject* obj, const upb_EnumDef* e, + upb_MessageValue* val) { + if (PyUnicode_Check(obj)) { + Py_ssize_t size; + const char* name = PyUnicode_AsUTF8AndSize(obj, &size); + const upb_EnumValueDef* ev = + upb_EnumDef_FindValueByNameWithSize(e, name, size); + if (!ev) { + PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", name); + return false; + } + val->int32_val = upb_EnumValueDef_Number(ev); + return true; + } else { + int32_t i32; + if (!PyUpb_GetInt32(obj, &i32)) return false; + if (upb_FileDef_Syntax(upb_EnumDef_File(e)) == kUpb_Syntax_Proto2 && + !upb_EnumDef_CheckNumber(e, i32)) { + PyErr_Format(PyExc_ValueError, "invalid enumerator %d", (int)i32); + return false; + } + val->int32_val = i32; + return true; + } +} + +bool PyUpb_IsNumpyNdarray(PyObject* obj, const upb_FieldDef* f) { + PyObject* type_name_obj = + PyObject_GetAttrString((PyObject*)Py_TYPE(obj), "__name__"); + bool is_ndarray = false; + if (!strcmp(PyUpb_GetStrData(type_name_obj), "ndarray")) { + PyErr_Format(PyExc_TypeError, + "%S has type ndarray, but expected one of: %s", obj, + upb_FieldDef_TypeString(f)); + is_ndarray = true; + } + Py_DECREF(type_name_obj); + return is_ndarray; +} + +bool PyUpb_PyToUpb(PyObject* obj, const upb_FieldDef* f, upb_MessageValue* val, + upb_Arena* arena) { + switch (upb_FieldDef_CType(f)) { + case kUpb_CType_Enum: + return PyUpb_PyToUpbEnum(obj, upb_FieldDef_EnumSubDef(f), val); + case kUpb_CType_Int32: + return PyUpb_GetInt32(obj, &val->int32_val); + case kUpb_CType_Int64: + return PyUpb_GetInt64(obj, &val->int64_val); + case kUpb_CType_UInt32: + return PyUpb_GetUint32(obj, &val->uint32_val); + case kUpb_CType_UInt64: + return PyUpb_GetUint64(obj, &val->uint64_val); + case kUpb_CType_Float: + if (PyUpb_IsNumpyNdarray(obj, f)) return false; + val->float_val = PyFloat_AsDouble(obj); + return !PyErr_Occurred(); + case kUpb_CType_Double: + if (PyUpb_IsNumpyNdarray(obj, f)) return false; + val->double_val = PyFloat_AsDouble(obj); + return !PyErr_Occurred(); + case kUpb_CType_Bool: + if (PyUpb_IsNumpyNdarray(obj, f)) return false; + val->bool_val = PyLong_AsLong(obj); + return !PyErr_Occurred(); + case kUpb_CType_Bytes: { + char* ptr; + Py_ssize_t size; + if (PyBytes_AsStringAndSize(obj, &ptr, &size) < 0) return false; + *val = PyUpb_MaybeCopyString(ptr, size, arena); + return true; + } + case kUpb_CType_String: { + Py_ssize_t size; + const char* ptr; + PyObject* unicode = NULL; + if (PyBytes_Check(obj)) { + unicode = obj = PyUnicode_FromEncodedObject(obj, "utf-8", NULL); + if (!obj) return false; + } + ptr = PyUnicode_AsUTF8AndSize(obj, &size); + if (PyErr_Occurred()) { + Py_XDECREF(unicode); + return false; + } + *val = PyUpb_MaybeCopyString(ptr, size, arena); + Py_XDECREF(unicode); + return true; + } + case kUpb_CType_Message: + PyErr_Format(PyExc_ValueError, "Message objects may not be assigned"); + return false; + default: + PyErr_Format(PyExc_SystemError, + "Getting a value from a field of unknown type %d", + upb_FieldDef_CType(f)); + return false; + } +} + +bool upb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2, + const upb_MessageDef* m); + +// ----------------------------------------------------------------------------- +// Equal +// ----------------------------------------------------------------------------- + +bool PyUpb_ValueEq(upb_MessageValue val1, upb_MessageValue val2, + const upb_FieldDef* f) { + switch (upb_FieldDef_CType(f)) { + case kUpb_CType_Bool: + return val1.bool_val == val2.bool_val; + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return val1.int32_val == val2.int32_val; + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return val1.int64_val == val2.int64_val; + case kUpb_CType_Float: + return val1.float_val == val2.float_val; + case kUpb_CType_Double: + return val1.double_val == val2.double_val; + case kUpb_CType_String: + case kUpb_CType_Bytes: + return val1.str_val.size == val2.str_val.size && + memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == + 0; + case kUpb_CType_Message: + return upb_Message_IsEqual(val1.msg_val, val2.msg_val, + upb_FieldDef_MessageSubDef(f)); + default: + return false; + } +} + +bool PyUpb_Map_IsEqual(const upb_Map* map1, const upb_Map* map2, + const upb_FieldDef* f) { + assert(upb_FieldDef_IsMap(f)); + if (map1 == map2) return true; + + size_t size1 = map1 ? upb_Map_Size(map1) : 0; + size_t size2 = map2 ? upb_Map_Size(map2) : 0; + if (size1 != size2) return false; + if (size1 == 0) return true; + + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + size_t iter = kUpb_Map_Begin; + + upb_MessageValue key, val1; + while (upb_Map_Next(map1, &key, &val1, &iter)) { + upb_MessageValue val2; + if (!upb_Map_Get(map2, key, &val2)) return false; + if (!PyUpb_ValueEq(val1, val2, val_f)) return false; + } + + return true; +} + +static bool PyUpb_ArrayElem_IsEqual(const upb_Array* arr1, + const upb_Array* arr2, size_t i, + const upb_FieldDef* f) { + assert(i < upb_Array_Size(arr1)); + assert(i < upb_Array_Size(arr2)); + upb_MessageValue val1 = upb_Array_Get(arr1, i); + upb_MessageValue val2 = upb_Array_Get(arr2, i); + return PyUpb_ValueEq(val1, val2, f); +} + +bool PyUpb_Array_IsEqual(const upb_Array* arr1, const upb_Array* arr2, + const upb_FieldDef* f) { + assert(upb_FieldDef_IsRepeated(f) && !upb_FieldDef_IsMap(f)); + if (arr1 == arr2) return true; + + size_t n1 = arr1 ? upb_Array_Size(arr1) : 0; + size_t n2 = arr2 ? upb_Array_Size(arr2) : 0; + if (n1 != n2) return false; + + // Half the length rounded down. Important: the empty list rounds to 0. + size_t half = n1 / 2; + + // Search from the ends-in. We expect differences to more quickly manifest + // at the ends than in the middle. If the length is odd we will miss the + // middle element. + for (size_t i = 0; i < half; i++) { + if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, i, f)) return false; + if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, n1 - 1 - i, f)) return false; + } + + // For an odd-lengthed list, pick up the middle element. + if (n1 & 1) { + if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, half, f)) return false; + } + + return true; +} + +bool upb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2, + const upb_MessageDef* m) { + if (msg1 == msg2) return true; + if (upb_Message_ExtensionCount(msg1) != upb_Message_ExtensionCount(msg2)) + return false; + + // Compare messages field-by-field. This is slightly tricky, because while + // we can iterate over normal fields in a predictable order, the extension + // order is unpredictable and may be different between msg1 and msg2. + // So we use the following strategy: + // 1. Iterate over all msg1 fields (including extensions). + // 2. For non-extension fields, we find the corresponding field by simply + // using upb_Message_Next(msg2). If the two messages have the same set + // of fields, this will yield the same field. + // 3. For extension fields, we have to actually search for the corresponding + // field, which we do with upb_Message_GetFieldByDef(msg2, ext_f1). + // 4. Once iteration over msg1 is complete, we call upb_Message_Next(msg2) + // one + // final time to verify that we have visited all of msg2's regular fields + // (we pass NULL for ext_dict so that iteration will *not* return + // extensions). + // + // We don't need to visit all of msg2's extensions, because we verified up + // front that both messages have the same number of extensions. + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m)); + const upb_FieldDef *f1, *f2; + upb_MessageValue val1, val2; + size_t iter1 = kUpb_Message_Begin; + size_t iter2 = kUpb_Message_Begin; + while (upb_Message_Next(msg1, m, symtab, &f1, &val1, &iter1)) { + if (upb_FieldDef_IsExtension(f1)) { + val2 = upb_Message_GetFieldByDef(msg2, f1); + } else { + if (!upb_Message_Next(msg2, m, NULL, &f2, &val2, &iter2) || f1 != f2) { + return false; + } + } + + if (upb_FieldDef_IsMap(f1)) { + if (!PyUpb_Map_IsEqual(val1.map_val, val2.map_val, f1)) return false; + } else if (upb_FieldDef_IsRepeated(f1)) { + if (!PyUpb_Array_IsEqual(val1.array_val, val2.array_val, f1)) { + return false; + } + } else { + if (!PyUpb_ValueEq(val1, val2, f1)) return false; + } + } + + if (upb_Message_Next(msg2, m, NULL, &f2, &val2, &iter2)) return false; + + size_t usize1, usize2; + const char* uf1 = upb_Message_GetUnknown(msg1, &usize1); + const char* uf2 = upb_Message_GetUnknown(msg2, &usize2); + // 100 is arbitrary, we're trying to prevent stack overflow but it's not + // obvious how deep we should allow here. + return upb_Message_UnknownFieldsAreEqual(uf1, usize1, uf2, usize2, 100) == + kUpb_UnknownCompareResult_Equal; +} + +#include "upb/port/undef.inc"
diff --git a/python/convert.h b/python/convert.h new file mode 100644 index 0000000..1c594d3 --- /dev/null +++ b/python/convert.h
@@ -0,0 +1,66 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_CONVERT_H__ +#define PYUPB_CONVERT_H__ + +#include "protobuf.h" +#include "upb/reflection/def.h" +#include "upb/reflection/message.h" + +// Converts `val` to a Python object according to the type information in `f`. +// Any newly-created Python objects that reference non-primitive data from `val` +// will take a reference on `arena`; the caller must ensure that `val` belongs +// to `arena`. If the conversion cannot be performed, returns NULL and sets a +// Python error. +PyObject* PyUpb_UpbToPy(upb_MessageValue val, const upb_FieldDef* f, + PyObject* arena); + +// Converts `obj` to a upb_MessageValue `*val` according to the type information +// in `f`. If `arena` is provided, any string data will be copied into `arena`, +// otherwise the returned value will alias the Python-owned data (this can be +// useful for an ephemeral upb_MessageValue). If the conversion cannot be +// performed, returns false. +bool PyUpb_PyToUpb(PyObject* obj, const upb_FieldDef* f, upb_MessageValue* val, + upb_Arena* arena); + +// Returns true if the given values (of type `f`) are equal. +bool PyUpb_ValueEq(upb_MessageValue val1, upb_MessageValue val2, + const upb_FieldDef* f); + +// Returns true if the two arrays (with element type `f`) are equal. +bool PyUpb_Array_IsEqual(const upb_Array* arr1, const upb_Array* arr2, + const upb_FieldDef* f); + +// Returns true if the given messages (of type `m`) are equal. +bool upb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2, + const upb_MessageDef* m); + +#endif // PYUPB_CONVERT_H__
diff --git a/python/descriptor.c b/python/descriptor.c new file mode 100644 index 0000000..d1726e4 --- /dev/null +++ b/python/descriptor.c
@@ -0,0 +1,1770 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/descriptor.h" + +#include "python/convert.h" +#include "python/descriptor_containers.h" +#include "python/descriptor_pool.h" +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/reflection/def.h" +#include "upb/util/def_to_proto.h" + +// ----------------------------------------------------------------------------- +// DescriptorBase +// ----------------------------------------------------------------------------- + +// This representation is used by all concrete descriptors. + +typedef struct { + PyObject_HEAD; + PyObject* pool; // We own a ref. + const void* def; // Type depends on the class. Kept alive by "pool". + PyObject* options; // NULL if not present or not cached. + PyObject* message_meta; // We own a ref. +} PyUpb_DescriptorBase; + +PyObject* PyUpb_AnyDescriptor_GetPool(PyObject* desc) { + PyUpb_DescriptorBase* base = (void*)desc; + return base->pool; +} + +const void* PyUpb_AnyDescriptor_GetDef(PyObject* desc) { + PyUpb_DescriptorBase* base = (void*)desc; + return base->def; +} + +static PyUpb_DescriptorBase* PyUpb_DescriptorBase_DoCreate( + PyUpb_DescriptorType type, const void* def, const upb_FileDef* file) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyTypeObject* type_obj = state->descriptor_types[type]; + assert(def); + + PyUpb_DescriptorBase* base = (void*)PyType_GenericAlloc(type_obj, 0); + base->pool = PyUpb_DescriptorPool_Get(upb_FileDef_Pool(file)); + base->def = def; + base->options = NULL; + base->message_meta = NULL; + + PyUpb_ObjCache_Add(def, &base->ob_base); + return base; +} + +// Returns a Python object wrapping |def|, of descriptor type |type|. If a +// wrapper was previously created for this def, returns it, otherwise creates a +// new wrapper. +static PyObject* PyUpb_DescriptorBase_Get(PyUpb_DescriptorType type, + const void* def, + const upb_FileDef* file) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)PyUpb_ObjCache_Get(def); + + if (!base) { + base = PyUpb_DescriptorBase_DoCreate(type, def, file); + } + + return &base->ob_base; +} + +static PyUpb_DescriptorBase* PyUpb_DescriptorBase_Check( + PyObject* obj, PyUpb_DescriptorType type) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyTypeObject* type_obj = state->descriptor_types[type]; + if (!PyObject_TypeCheck(obj, type_obj)) { + PyErr_Format(PyExc_TypeError, "Expected object of type %S, but got %R", + type_obj, obj); + return NULL; + } + return (PyUpb_DescriptorBase*)obj; +} + +static PyObject* PyUpb_DescriptorBase_GetOptions(PyUpb_DescriptorBase* self, + const upb_Message* opts, + const upb_MiniTable* layout, + const char* msg_name) { + if (!self->options) { + // Load descriptors protos if they are not loaded already. We have to do + // this lazily, otherwise, it would lead to circular imports. + PyObject* mod = PyImport_ImportModuleLevel(PYUPB_DESCRIPTOR_MODULE, NULL, + NULL, NULL, 0); + if (mod == NULL) return NULL; + Py_DECREF(mod); + + // Find the correct options message. + PyObject* default_pool = PyUpb_DescriptorPool_GetDefaultPool(); + const upb_DefPool* symtab = PyUpb_DescriptorPool_GetSymtab(default_pool); + const upb_MessageDef* m = upb_DefPool_FindMessageByName(symtab, msg_name); + assert(m); + + // Copy the options message from C to Python using serialize+parse. + // We don't wrap the C object directly because there is no guarantee that + // the descriptor_pb2 that was loaded at runtime has the same members or + // layout as the C types that were compiled in. + size_t size; + PyObject* py_arena = PyUpb_Arena_New(); + upb_Arena* arena = PyUpb_Arena_Get(py_arena); + char* pb; + // TODO: Need to correctly handle failed return codes. + (void)upb_Encode(opts, layout, 0, arena, &pb, &size); + const upb_MiniTable* opts2_layout = upb_MessageDef_MiniTable(m); + upb_Message* opts2 = upb_Message_New(opts2_layout, arena); + assert(opts2); + upb_DecodeStatus ds = + upb_Decode(pb, size, opts2, opts2_layout, + upb_DefPool_ExtensionRegistry(symtab), 0, arena); + (void)ds; + assert(ds == kUpb_DecodeStatus_Ok); + + self->options = PyUpb_Message_Get(opts2, m, py_arena); + Py_DECREF(py_arena); + } + + Py_INCREF(self->options); + return self->options; +} + +typedef void* PyUpb_ToProto_Func(const void* def, upb_Arena* arena); + +static PyObject* PyUpb_DescriptorBase_GetSerializedProto( + PyObject* _self, PyUpb_ToProto_Func* func, const upb_MiniTable* layout) { + PyUpb_DescriptorBase* self = (void*)_self; + upb_Arena* arena = upb_Arena_New(); + if (!arena) PYUPB_RETURN_OOM; + upb_Message* proto = func(self->def, arena); + if (!proto) goto oom; + size_t size; + char* pb; + upb_EncodeStatus status = upb_Encode(proto, layout, 0, arena, &pb, &size); + if (status) goto oom; // TODO non-oom errors are possible here + PyObject* str = PyBytes_FromStringAndSize(pb, size); + upb_Arena_Free(arena); + return str; + +oom: + upb_Arena_Free(arena); + PyErr_SetNone(PyExc_MemoryError); + return NULL; +} + +static PyObject* PyUpb_DescriptorBase_CopyToProto(PyObject* _self, + PyUpb_ToProto_Func* func, + const upb_MiniTable* layout, + const char* expected_type, + PyObject* py_proto) { + if (!PyUpb_Message_Verify(py_proto)) return NULL; + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(py_proto); + const char* type = upb_MessageDef_FullName(m); + if (strcmp(type, expected_type) != 0) { + PyErr_Format( + PyExc_TypeError, + "CopyToProto: message is of incorrect type '%s' (expected '%s'", type, + expected_type); + return NULL; + } + PyObject* serialized = + PyUpb_DescriptorBase_GetSerializedProto(_self, func, layout); + if (!serialized) return NULL; + PyObject* ret = PyUpb_Message_MergeFromString(py_proto, serialized); + Py_DECREF(serialized); + return ret; +} + +static void PyUpb_DescriptorBase_Dealloc(PyUpb_DescriptorBase* base) { + PyUpb_ObjCache_Delete(base->def); + Py_XDECREF(base->message_meta); + Py_DECREF(base->pool); + Py_XDECREF(base->options); + PyUpb_Dealloc(base); +} + +static int PyUpb_Descriptor_Traverse(PyUpb_DescriptorBase* base, + visitproc visit, void* arg) { + Py_VISIT(base->message_meta); + return 0; +} + +static int PyUpb_Descriptor_Clear(PyUpb_DescriptorBase* base) { + Py_CLEAR(base->message_meta); + return 0; +} + +#define DESCRIPTOR_BASE_SLOTS \ + {Py_tp_new, (void*)&PyUpb_Forbidden_New}, { \ + Py_tp_dealloc, (void*)&PyUpb_DescriptorBase_Dealloc \ + } + +// ----------------------------------------------------------------------------- +// Descriptor +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_Descriptor_Get(const upb_MessageDef* m) { + assert(m); + const upb_FileDef* file = upb_MessageDef_File(m); + return PyUpb_DescriptorBase_Get(kPyUpb_Descriptor, m, file); +} + +PyObject* PyUpb_Descriptor_GetClass(const upb_MessageDef* m) { + PyObject* ret = PyUpb_ObjCache_Get(upb_MessageDef_MiniTable(m)); + if (ret) return ret; + + // On demand create the clss if not exist. However, if users repeatedly + // create and destroy a class, it could trigger a loop. This is not an + // issue now, but if we see CPU waste for repeatedly create and destroy + // in the future, we could make PyUpb_Descriptor_Get() append the descriptor + // to an internal list in DescriptorPool, let the pool keep descriptors alive. + PyObject* py_descriptor = PyUpb_Descriptor_Get(m); + if (py_descriptor == NULL) return NULL; + const char* name = upb_MessageDef_Name(m); + PyObject* dict = PyDict_New(); + if (dict == NULL) goto err; + int status = PyDict_SetItemString(dict, "DESCRIPTOR", py_descriptor); + if (status < 0) goto err; + ret = PyUpb_MessageMeta_DoCreateClass(py_descriptor, name, dict); + +err: + Py_XDECREF(py_descriptor); + Py_XDECREF(dict); + return ret; +} + +void PyUpb_Descriptor_SetClass(PyObject* py_descriptor, PyObject* meta) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)py_descriptor; + Py_XDECREF(base->message_meta); + base->message_meta = meta; + Py_INCREF(meta); +} + +// The LookupNested*() functions provide name lookup for entities nested inside +// a message. This uses the symtab's table, which requires that the symtab is +// not being mutated concurrently. We can guarantee this for Python-owned +// symtabs, but upb cannot guarantee it in general for an arbitrary +// `const upb_MessageDef*`. + +static const void* PyUpb_Descriptor_LookupNestedMessage(const upb_MessageDef* m, + const char* name) { + const upb_FileDef* filedef = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(filedef); + PyObject* qname = + PyUnicode_FromFormat("%s.%s", upb_MessageDef_FullName(m), name); + const upb_MessageDef* ret = upb_DefPool_FindMessageByName( + symtab, PyUnicode_AsUTF8AndSize(qname, NULL)); + Py_DECREF(qname); + return ret; +} + +static const void* PyUpb_Descriptor_LookupNestedEnum(const upb_MessageDef* m, + const char* name) { + const upb_FileDef* filedef = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(filedef); + PyObject* qname = + PyUnicode_FromFormat("%s.%s", upb_MessageDef_FullName(m), name); + const upb_EnumDef* ret = + upb_DefPool_FindEnumByName(symtab, PyUnicode_AsUTF8AndSize(qname, NULL)); + Py_DECREF(qname); + return ret; +} + +static const void* PyUpb_Descriptor_LookupNestedExtension( + const upb_MessageDef* m, const char* name) { + const upb_FileDef* filedef = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(filedef); + PyObject* qname = + PyUnicode_FromFormat("%s.%s", upb_MessageDef_FullName(m), name); + const upb_FieldDef* ret = upb_DefPool_FindExtensionByName( + symtab, PyUnicode_AsUTF8AndSize(qname, NULL)); + Py_DECREF(qname); + return ret; +} + +static PyObject* PyUpb_Descriptor_GetExtensionRanges(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (PyUpb_DescriptorBase*)_self; + int n = upb_MessageDef_ExtensionRangeCount(self->def); + PyObject* range_list = PyList_New(n); + + for (int i = 0; i < n; i++) { + const upb_ExtensionRange* range = + upb_MessageDef_ExtensionRange(self->def, i); + PyObject* start = PyLong_FromLong(upb_ExtensionRange_Start(range)); + PyObject* end = PyLong_FromLong(upb_ExtensionRange_End(range)); + PyList_SetItem(range_list, i, PyTuple_Pack(2, start, end)); + } + + return range_list; +} + +static PyObject* PyUpb_Descriptor_GetExtensions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_MessageDef_NestedExtensionCount, + (void*)&upb_MessageDef_NestedExtension, + (void*)&PyUpb_FieldDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetExtensionsByName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_NestedExtensionCount, + (void*)&upb_MessageDef_NestedExtension, + (void*)&PyUpb_FieldDescriptor_Get, + }, + (void*)&PyUpb_Descriptor_LookupNestedExtension, + (void*)&upb_FieldDef_Name, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetEnumTypes(PyObject* _self, void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_MessageDef_NestedEnumCount, + (void*)&upb_MessageDef_NestedEnum, + (void*)&PyUpb_EnumDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetOneofs(PyObject* _self, void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_MessageDef_OneofCount, + (void*)&upb_MessageDef_Oneof, + (void*)&PyUpb_OneofDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetOptions(PyObject* _self, PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_MessageDef_Options(self->def), &google_protobuf_MessageOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".MessageOptions"); +} + +static PyObject* PyUpb_Descriptor_CopyToProto(PyObject* _self, + PyObject* py_proto) { + return PyUpb_DescriptorBase_CopyToProto( + _self, (PyUpb_ToProto_Func*)&upb_MessageDef_ToProto, + &google_protobuf_DescriptorProto_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".DescriptorProto", py_proto); +} + +static PyObject* PyUpb_Descriptor_EnumValueName(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + const char* enum_name; + int number; + if (!PyArg_ParseTuple(args, "si", &enum_name, &number)) return NULL; + const upb_EnumDef* e = + PyUpb_Descriptor_LookupNestedEnum(self->def, enum_name); + if (!e) { + PyErr_SetString(PyExc_KeyError, enum_name); + return NULL; + } + const upb_EnumValueDef* ev = upb_EnumDef_FindValueByNumber(e, number); + if (!ev) { + PyErr_Format(PyExc_KeyError, "%d", number); + return NULL; + } + return PyUnicode_FromString(upb_EnumValueDef_Name(ev)); +} + +static PyObject* PyUpb_Descriptor_GetFieldsByName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_FieldCount, + (void*)&upb_MessageDef_Field, + (void*)&PyUpb_FieldDescriptor_Get, + }, + (void*)&upb_MessageDef_FindFieldByName, + (void*)&upb_FieldDef_Name, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetFieldsByCamelCaseName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_FieldCount, + (void*)&upb_MessageDef_Field, + (void*)&PyUpb_FieldDescriptor_Get, + }, + (void*)&upb_MessageDef_FindByJsonName, + (void*)&upb_FieldDef_JsonName, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetFieldsByNumber(PyObject* _self, + void* closure) { + static PyUpb_ByNumberMap_Funcs funcs = { + { + (void*)&upb_MessageDef_FieldCount, + (void*)&upb_MessageDef_Field, + (void*)&PyUpb_FieldDescriptor_Get, + }, + (void*)&upb_MessageDef_FindFieldByNumber, + (void*)&upb_FieldDef_Number, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNumberMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetNestedTypes(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_MessageDef_NestedMessageCount, + (void*)&upb_MessageDef_NestedMessage, + (void*)&PyUpb_Descriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetNestedTypesByName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_NestedMessageCount, + (void*)&upb_MessageDef_NestedMessage, + (void*)&PyUpb_Descriptor_Get, + }, + (void*)&PyUpb_Descriptor_LookupNestedMessage, + (void*)&upb_MessageDef_Name, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetContainingType(PyObject* _self, + void* closure) { + // upb does not natively store the lexical parent of a message type, but we + // can derive it with some string manipulation and a lookup. + PyUpb_DescriptorBase* self = (void*)_self; + const upb_MessageDef* m = self->def; + const upb_FileDef* file = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(file); + const char* full_name = upb_MessageDef_FullName(m); + const char* last_dot = strrchr(full_name, '.'); + if (!last_dot) Py_RETURN_NONE; + const upb_MessageDef* parent = upb_DefPool_FindMessageByNameWithSize( + symtab, full_name, last_dot - full_name); + if (!parent) Py_RETURN_NONE; + return PyUpb_Descriptor_Get(parent); +} + +static PyObject* PyUpb_Descriptor_GetEnumTypesByName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_NestedEnumCount, + (void*)&upb_MessageDef_NestedEnum, + (void*)&PyUpb_EnumDescriptor_Get, + }, + (void*)&PyUpb_Descriptor_LookupNestedEnum, + (void*)&upb_EnumDef_Name, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetIsExtendable(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + if (upb_MessageDef_ExtensionRangeCount(self->def) > 0) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +static PyObject* PyUpb_Descriptor_GetFullName(PyObject* self, void* closure) { + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(self); + return PyUnicode_FromString(upb_MessageDef_FullName(msgdef)); +} + +static PyObject* PyUpb_Descriptor_GetConcreteClass(PyObject* self, + void* closure) { + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(self); + return PyUpb_ObjCache_Get(upb_MessageDef_MiniTable(msgdef)); +} + +static PyObject* PyUpb_Descriptor_GetFile(PyObject* self, void* closure) { + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(self); + return PyUpb_FileDescriptor_Get(upb_MessageDef_File(msgdef)); +} + +static PyObject* PyUpb_Descriptor_GetFields(PyObject* _self, void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_MessageDef_FieldCount, + (void*)&upb_MessageDef_Field, + (void*)&PyUpb_FieldDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetHasOptions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_MessageDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_Descriptor_GetName(PyObject* self, void* closure) { + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(self); + return PyUnicode_FromString(upb_MessageDef_Name(msgdef)); +} + +static PyObject* PyUpb_Descriptor_GetEnumValuesByName(PyObject* _self, + void* closure) { + // upb does not natively store any table containing all nested values. + // Consider: + // message M { + // enum E1 { + // A = 0; + // B = 1; + // } + // enum E2 { + // C = 0; + // D = 1; + // } + // } + // + // In this case, upb stores tables for E1 and E2, but it does not store a + // table for M that combines them (it is rarely needed and costs precious + // space and time to build). + // + // To work around this, we build an actual Python dict whenever a user + // actually asks for this. + PyUpb_DescriptorBase* self = (void*)_self; + PyObject* ret = PyDict_New(); + if (!ret) return NULL; + int enum_count = upb_MessageDef_NestedEnumCount(self->def); + for (int i = 0; i < enum_count; i++) { + const upb_EnumDef* e = upb_MessageDef_NestedEnum(self->def, i); + int value_count = upb_EnumDef_ValueCount(e); + for (int j = 0; j < value_count; j++) { + // Collisions should be impossible here, as uniqueness is checked by + // protoc (this is an invariant of the protobuf language). However this + // uniqueness constraint is not currently checked by upb/def.c at load + // time, so if the user supplies a manually-constructed descriptor that + // does not respect this constraint, a collision could be possible and the + // last-defined enumerator would win. This could be seen as an argument + // for having upb actually build the table at load time, thus checking the + // constraint proactively, but upb is always checking a subset of the full + // validation performed by C++, and we have to pick and choose the biggest + // bang for the buck. + const upb_EnumValueDef* ev = upb_EnumDef_Value(e, j); + const char* name = upb_EnumValueDef_Name(ev); + PyObject* val = PyUpb_EnumValueDescriptor_Get(ev); + if (!val || PyDict_SetItemString(ret, name, val) < 0) { + Py_XDECREF(val); + Py_DECREF(ret); + return NULL; + } + Py_DECREF(val); + } + } + return ret; +} + +static PyObject* PyUpb_Descriptor_GetOneofsByName(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_MessageDef_OneofCount, + (void*)&upb_MessageDef_Oneof, + (void*)&PyUpb_OneofDescriptor_Get, + }, + (void*)&upb_MessageDef_FindOneofByName, + (void*)&upb_OneofDef_Name, + }; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_Descriptor_GetSyntax(PyObject* self, void* closure) { + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(self); + const char* syntax = + upb_MessageDef_Syntax(msgdef) == kUpb_Syntax_Proto2 ? "proto2" : "proto3"; + return PyUnicode_InternFromString(syntax); +} + +static PyGetSetDef PyUpb_Descriptor_Getters[] = { + {"name", PyUpb_Descriptor_GetName, NULL, "Last name"}, + {"full_name", PyUpb_Descriptor_GetFullName, NULL, "Full name"}, + {"_concrete_class", PyUpb_Descriptor_GetConcreteClass, NULL, + "concrete class"}, + {"file", PyUpb_Descriptor_GetFile, NULL, "File descriptor"}, + {"fields", PyUpb_Descriptor_GetFields, NULL, "Fields sequence"}, + {"fields_by_name", PyUpb_Descriptor_GetFieldsByName, NULL, + "Fields by name"}, + {"fields_by_camelcase_name", PyUpb_Descriptor_GetFieldsByCamelCaseName, + NULL, "Fields by camelCase name"}, + {"fields_by_number", PyUpb_Descriptor_GetFieldsByNumber, NULL, + "Fields by number"}, + {"nested_types", PyUpb_Descriptor_GetNestedTypes, NULL, + "Nested types sequence"}, + {"nested_types_by_name", PyUpb_Descriptor_GetNestedTypesByName, NULL, + "Nested types by name"}, + {"extensions", PyUpb_Descriptor_GetExtensions, NULL, "Extensions Sequence"}, + {"extensions_by_name", PyUpb_Descriptor_GetExtensionsByName, NULL, + "Extensions by name"}, + {"extension_ranges", PyUpb_Descriptor_GetExtensionRanges, NULL, + "Extension ranges"}, + {"enum_types", PyUpb_Descriptor_GetEnumTypes, NULL, "Enum sequence"}, + {"enum_types_by_name", PyUpb_Descriptor_GetEnumTypesByName, NULL, + "Enum types by name"}, + {"enum_values_by_name", PyUpb_Descriptor_GetEnumValuesByName, NULL, + "Enum values by name"}, + {"oneofs_by_name", PyUpb_Descriptor_GetOneofsByName, NULL, + "Oneofs by name"}, + {"oneofs", PyUpb_Descriptor_GetOneofs, NULL, "Oneofs Sequence"}, + {"containing_type", PyUpb_Descriptor_GetContainingType, NULL, + "Containing type"}, + {"is_extendable", PyUpb_Descriptor_GetIsExtendable, NULL}, + {"has_options", PyUpb_Descriptor_GetHasOptions, NULL, "Has Options"}, + // begin:github_only + {"syntax", &PyUpb_Descriptor_GetSyntax, NULL, "Syntax"}, + // end:github_only + // begin:google_only +// // TODO Use this to open-source syntax deprecation. +// {"deprecated_syntax", &PyUpb_Descriptor_GetSyntax, NULL, "Syntax"}, + // end:google_only + {NULL}}; + +static PyMethodDef PyUpb_Descriptor_Methods[] = { + {"GetOptions", PyUpb_Descriptor_GetOptions, METH_NOARGS}, + {"CopyToProto", PyUpb_Descriptor_CopyToProto, METH_O}, + {"EnumValueName", PyUpb_Descriptor_EnumValueName, METH_VARARGS}, + {NULL}}; + +static PyType_Slot PyUpb_Descriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_Descriptor_Methods}, + {Py_tp_getset, PyUpb_Descriptor_Getters}, + {Py_tp_traverse, PyUpb_Descriptor_Traverse}, + {Py_tp_clear, PyUpb_Descriptor_Clear}, + {0, NULL}}; + +static PyType_Spec PyUpb_Descriptor_Spec = { + PYUPB_MODULE_NAME ".Descriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, // tp_flags + PyUpb_Descriptor_Slots, +}; + +const upb_MessageDef* PyUpb_Descriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_Descriptor); + return self ? self->def : NULL; +} + +// ----------------------------------------------------------------------------- +// EnumDescriptor +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_EnumDescriptor_Get(const upb_EnumDef* enumdef) { + const upb_FileDef* file = upb_EnumDef_File(enumdef); + return PyUpb_DescriptorBase_Get(kPyUpb_EnumDescriptor, enumdef, file); +} + +const upb_EnumDef* PyUpb_EnumDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_EnumDescriptor); + return self ? self->def : NULL; +} + +static PyObject* PyUpb_EnumDescriptor_GetFullName(PyObject* self, + void* closure) { + const upb_EnumDef* enumdef = PyUpb_EnumDescriptor_GetDef(self); + return PyUnicode_FromString(upb_EnumDef_FullName(enumdef)); +} + +static PyObject* PyUpb_EnumDescriptor_GetName(PyObject* self, void* closure) { + const upb_EnumDef* enumdef = PyUpb_EnumDescriptor_GetDef(self); + return PyUnicode_FromString(upb_EnumDef_Name(enumdef)); +} + +static PyObject* PyUpb_EnumDescriptor_GetFile(PyObject* self, void* closure) { + const upb_EnumDef* enumdef = PyUpb_EnumDescriptor_GetDef(self); + return PyUpb_FileDescriptor_Get(upb_EnumDef_File(enumdef)); +} + +static PyObject* PyUpb_EnumDescriptor_GetValues(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_EnumDef_ValueCount, + (void*)&upb_EnumDef_Value, + (void*)&PyUpb_EnumValueDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_EnumDescriptor_GetValuesByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_EnumDef_ValueCount, + (void*)&upb_EnumDef_Value, + (void*)&PyUpb_EnumValueDescriptor_Get, + }, + (void*)&upb_EnumDef_FindValueByName, + (void*)&upb_EnumValueDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_EnumDescriptor_GetValuesByNumber(PyObject* _self, + void* closure) { + static PyUpb_ByNumberMap_Funcs funcs = { + { + (void*)&upb_EnumDef_ValueCount, + (void*)&upb_EnumDef_Value, + (void*)&PyUpb_EnumValueDescriptor_Get, + }, + (void*)&upb_EnumDef_FindValueByNumber, + (void*)&upb_EnumValueDef_Number, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNumberMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_EnumDescriptor_GetContainingType(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + const upb_MessageDef* m = upb_EnumDef_ContainingType(self->def); + if (!m) Py_RETURN_NONE; + return PyUpb_Descriptor_Get(m); +} + +static PyObject* PyUpb_EnumDescriptor_GetHasOptions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_EnumDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_EnumDescriptor_GetIsClosed(PyObject* _self, + void* closure) { + const upb_EnumDef* enumdef = PyUpb_EnumDescriptor_GetDef(_self); + return PyBool_FromLong(upb_EnumDef_IsClosed(enumdef)); +} + +static PyObject* PyUpb_EnumDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_EnumDef_Options(self->def), &google_protobuf_EnumOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".EnumOptions"); +} + +static PyObject* PyUpb_EnumDescriptor_CopyToProto(PyObject* _self, + PyObject* py_proto) { + return PyUpb_DescriptorBase_CopyToProto( + _self, (PyUpb_ToProto_Func*)&upb_EnumDef_ToProto, + &google_protobuf_EnumDescriptorProto_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".EnumDescriptorProto", py_proto); +} + +static PyGetSetDef PyUpb_EnumDescriptor_Getters[] = { + {"full_name", PyUpb_EnumDescriptor_GetFullName, NULL, "Full name"}, + {"name", PyUpb_EnumDescriptor_GetName, NULL, "last name"}, + {"file", PyUpb_EnumDescriptor_GetFile, NULL, "File descriptor"}, + {"values", PyUpb_EnumDescriptor_GetValues, NULL, "values"}, + {"values_by_name", PyUpb_EnumDescriptor_GetValuesByName, NULL, + "Enum values by name"}, + {"values_by_number", PyUpb_EnumDescriptor_GetValuesByNumber, NULL, + "Enum values by number"}, + {"containing_type", PyUpb_EnumDescriptor_GetContainingType, NULL, + "Containing type"}, + {"has_options", PyUpb_EnumDescriptor_GetHasOptions, NULL, "Has Options"}, + {"is_closed", PyUpb_EnumDescriptor_GetIsClosed, NULL, + "Checks if the enum is closed"}, + {NULL}}; + +static PyMethodDef PyUpb_EnumDescriptor_Methods[] = { + {"GetOptions", PyUpb_EnumDescriptor_GetOptions, METH_NOARGS}, + {"CopyToProto", PyUpb_EnumDescriptor_CopyToProto, METH_O}, + {NULL}}; + +static PyType_Slot PyUpb_EnumDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_EnumDescriptor_Methods}, + {Py_tp_getset, PyUpb_EnumDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_EnumDescriptor_Spec = { + PYUPB_MODULE_NAME ".EnumDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_EnumDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// EnumValueDescriptor +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_EnumValueDescriptor_Get(const upb_EnumValueDef* ev) { + const upb_FileDef* file = upb_EnumDef_File(upb_EnumValueDef_Enum(ev)); + return PyUpb_DescriptorBase_Get(kPyUpb_EnumValueDescriptor, ev, file); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetName(PyObject* self, + void* closure) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)self; + return PyUnicode_FromString(upb_EnumValueDef_Name(base->def)); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetNumber(PyObject* self, + void* closure) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)self; + return PyLong_FromLong(upb_EnumValueDef_Number(base->def)); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetIndex(PyObject* self, + void* closure) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)self; + return PyLong_FromLong(upb_EnumValueDef_Index(base->def)); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetType(PyObject* self, + void* closure) { + PyUpb_DescriptorBase* base = (PyUpb_DescriptorBase*)self; + return PyUpb_EnumDescriptor_Get(upb_EnumValueDef_Enum(base->def)); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetHasOptions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_EnumValueDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_EnumValueDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_EnumValueDef_Options(self->def), + &google_protobuf_EnumValueOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".EnumValueOptions"); +} + +static PyGetSetDef PyUpb_EnumValueDescriptor_Getters[] = { + {"name", PyUpb_EnumValueDescriptor_GetName, NULL, "name"}, + {"number", PyUpb_EnumValueDescriptor_GetNumber, NULL, "number"}, + {"index", PyUpb_EnumValueDescriptor_GetIndex, NULL, "index"}, + {"type", PyUpb_EnumValueDescriptor_GetType, NULL, "index"}, + {"has_options", PyUpb_EnumValueDescriptor_GetHasOptions, NULL, + "Has Options"}, + {NULL}}; + +static PyMethodDef PyUpb_EnumValueDescriptor_Methods[] = { + { + "GetOptions", + PyUpb_EnumValueDescriptor_GetOptions, + METH_NOARGS, + }, + {NULL}}; + +static PyType_Slot PyUpb_EnumValueDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_EnumValueDescriptor_Methods}, + {Py_tp_getset, PyUpb_EnumValueDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_EnumValueDescriptor_Spec = { + PYUPB_MODULE_NAME ".EnumValueDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_EnumValueDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// FieldDescriptor +// ----------------------------------------------------------------------------- + +const upb_FieldDef* PyUpb_FieldDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_FieldDescriptor); + return self ? self->def : NULL; +} + +PyObject* PyUpb_FieldDescriptor_Get(const upb_FieldDef* field) { + const upb_FileDef* file = upb_FieldDef_File(field); + return PyUpb_DescriptorBase_Get(kPyUpb_FieldDescriptor, field, file); +} + +static PyObject* PyUpb_FieldDescriptor_GetFullName(PyUpb_DescriptorBase* self, + void* closure) { + return PyUnicode_FromString(upb_FieldDef_FullName(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetName(PyUpb_DescriptorBase* self, + void* closure) { + return PyUnicode_FromString(upb_FieldDef_Name(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetCamelCaseName( + PyUpb_DescriptorBase* self, void* closure) { + // TODO: Ok to use jsonname here? + return PyUnicode_FromString(upb_FieldDef_JsonName(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetJsonName(PyUpb_DescriptorBase* self, + void* closure) { + return PyUnicode_FromString(upb_FieldDef_JsonName(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetFile(PyUpb_DescriptorBase* self, + void* closure) { + const upb_FileDef* file = upb_FieldDef_File(self->def); + if (!file) Py_RETURN_NONE; + return PyUpb_FileDescriptor_Get(file); +} + +static PyObject* PyUpb_FieldDescriptor_GetType(PyUpb_DescriptorBase* self, + void* closure) { + return PyLong_FromLong(upb_FieldDef_Type(self->def)); +} + +// Enum values copied from descriptor.h in C++. +enum CppType { + CPPTYPE_INT32 = 1, // TYPE_INT32, TYPE_SINT32, TYPE_SFIXED32 + CPPTYPE_INT64 = 2, // TYPE_INT64, TYPE_SINT64, TYPE_SFIXED64 + CPPTYPE_UINT32 = 3, // TYPE_UINT32, TYPE_FIXED32 + CPPTYPE_UINT64 = 4, // TYPE_UINT64, TYPE_FIXED64 + CPPTYPE_DOUBLE = 5, // TYPE_DOUBLE + CPPTYPE_FLOAT = 6, // TYPE_FLOAT + CPPTYPE_BOOL = 7, // TYPE_BOOL + CPPTYPE_ENUM = 8, // TYPE_ENUM + CPPTYPE_STRING = 9, // TYPE_STRING, TYPE_BYTES + CPPTYPE_MESSAGE = 10, // TYPE_MESSAGE, TYPE_GROUP +}; + +static PyObject* PyUpb_FieldDescriptor_GetCppType(PyUpb_DescriptorBase* self, + void* closure) { + static const uint8_t cpp_types[] = { + -1, + [kUpb_CType_Int32] = CPPTYPE_INT32, + [kUpb_CType_Int64] = CPPTYPE_INT64, + [kUpb_CType_UInt32] = CPPTYPE_UINT32, + [kUpb_CType_UInt64] = CPPTYPE_UINT64, + [kUpb_CType_Double] = CPPTYPE_DOUBLE, + [kUpb_CType_Float] = CPPTYPE_FLOAT, + [kUpb_CType_Bool] = CPPTYPE_BOOL, + [kUpb_CType_Enum] = CPPTYPE_ENUM, + [kUpb_CType_String] = CPPTYPE_STRING, + [kUpb_CType_Bytes] = CPPTYPE_STRING, + [kUpb_CType_Message] = CPPTYPE_MESSAGE, + }; + return PyLong_FromLong(cpp_types[upb_FieldDef_CType(self->def)]); +} + +static PyObject* PyUpb_FieldDescriptor_GetLabel(PyUpb_DescriptorBase* self, + void* closure) { + return PyLong_FromLong(upb_FieldDef_Label(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetIsExtension( + PyUpb_DescriptorBase* self, void* closure) { + return PyBool_FromLong(upb_FieldDef_IsExtension(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetNumber(PyUpb_DescriptorBase* self, + void* closure) { + return PyLong_FromLong(upb_FieldDef_Number(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetIndex(PyUpb_DescriptorBase* self, + void* closure) { + return PyLong_FromLong(upb_FieldDef_Index(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetMessageType( + PyUpb_DescriptorBase* self, void* closure) { + const upb_MessageDef* subdef = upb_FieldDef_MessageSubDef(self->def); + if (!subdef) Py_RETURN_NONE; + return PyUpb_Descriptor_Get(subdef); +} + +static PyObject* PyUpb_FieldDescriptor_GetEnumType(PyUpb_DescriptorBase* self, + void* closure) { + const upb_EnumDef* enumdef = upb_FieldDef_EnumSubDef(self->def); + if (!enumdef) Py_RETURN_NONE; + return PyUpb_EnumDescriptor_Get(enumdef); +} + +static PyObject* PyUpb_FieldDescriptor_GetContainingType( + PyUpb_DescriptorBase* self, void* closure) { + const upb_MessageDef* m = upb_FieldDef_ContainingType(self->def); + if (!m) Py_RETURN_NONE; + return PyUpb_Descriptor_Get(m); +} + +static PyObject* PyUpb_FieldDescriptor_GetExtensionScope( + PyUpb_DescriptorBase* self, void* closure) { + const upb_MessageDef* m = upb_FieldDef_ExtensionScope(self->def); + if (!m) Py_RETURN_NONE; + return PyUpb_Descriptor_Get(m); +} + +static PyObject* PyUpb_FieldDescriptor_HasDefaultValue( + PyUpb_DescriptorBase* self, void* closure) { + return PyBool_FromLong(upb_FieldDef_HasDefault(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetDefaultValue( + PyUpb_DescriptorBase* self, void* closure) { + const upb_FieldDef* f = self->def; + if (upb_FieldDef_IsRepeated(f)) return PyList_New(0); + if (upb_FieldDef_IsSubMessage(f)) Py_RETURN_NONE; + return PyUpb_UpbToPy(upb_FieldDef_Default(self->def), self->def, NULL); +} + +static PyObject* PyUpb_FieldDescriptor_GetContainingOneof( + PyUpb_DescriptorBase* self, void* closure) { + const upb_OneofDef* oneof = upb_FieldDef_ContainingOneof(self->def); + if (!oneof) Py_RETURN_NONE; + return PyUpb_OneofDescriptor_Get(oneof); +} + +static PyObject* PyUpb_FieldDescriptor_GetHasOptions( + PyUpb_DescriptorBase* _self, void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_FieldDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetHasPresence( + PyUpb_DescriptorBase* _self, void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_FieldDef_HasPresence(self->def)); +} + +static PyObject* PyUpb_FieldDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_FieldDef_Options(self->def), &google_protobuf_FieldOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FieldOptions"); +} + +static PyGetSetDef PyUpb_FieldDescriptor_Getters[] = { + {"full_name", (getter)PyUpb_FieldDescriptor_GetFullName, NULL, "Full name"}, + {"name", (getter)PyUpb_FieldDescriptor_GetName, NULL, "Unqualified name"}, + {"camelcase_name", (getter)PyUpb_FieldDescriptor_GetCamelCaseName, NULL, + "CamelCase name"}, + {"json_name", (getter)PyUpb_FieldDescriptor_GetJsonName, NULL, "Json name"}, + {"file", (getter)PyUpb_FieldDescriptor_GetFile, NULL, "File Descriptor"}, + {"type", (getter)PyUpb_FieldDescriptor_GetType, NULL, "Type"}, + {"cpp_type", (getter)PyUpb_FieldDescriptor_GetCppType, NULL, "C++ Type"}, + {"label", (getter)PyUpb_FieldDescriptor_GetLabel, NULL, "Label"}, + {"number", (getter)PyUpb_FieldDescriptor_GetNumber, NULL, "Number"}, + {"index", (getter)PyUpb_FieldDescriptor_GetIndex, NULL, "Index"}, + {"default_value", (getter)PyUpb_FieldDescriptor_GetDefaultValue, NULL, + "Default Value"}, + {"has_default_value", (getter)PyUpb_FieldDescriptor_HasDefaultValue}, + {"is_extension", (getter)PyUpb_FieldDescriptor_GetIsExtension, NULL, "ID"}, + // TODO + //{ "id", (getter)GetID, NULL, "ID"}, + {"message_type", (getter)PyUpb_FieldDescriptor_GetMessageType, NULL, + "Message type"}, + {"enum_type", (getter)PyUpb_FieldDescriptor_GetEnumType, NULL, "Enum type"}, + {"containing_type", (getter)PyUpb_FieldDescriptor_GetContainingType, NULL, + "Containing type"}, + {"extension_scope", (getter)PyUpb_FieldDescriptor_GetExtensionScope, NULL, + "Extension scope"}, + {"containing_oneof", (getter)PyUpb_FieldDescriptor_GetContainingOneof, NULL, + "Containing oneof"}, + {"has_options", (getter)PyUpb_FieldDescriptor_GetHasOptions, NULL, + "Has Options"}, + {"has_presence", (getter)PyUpb_FieldDescriptor_GetHasPresence, NULL, + "Has Presence"}, + // TODO + //{ "_options", + //(getter)NULL, (setter)SetOptions, "Options"}, { "_serialized_options", + //(getter)NULL, (setter)SetSerializedOptions, "Serialized Options"}, + {NULL}}; + +static PyMethodDef PyUpb_FieldDescriptor_Methods[] = { + { + "GetOptions", + PyUpb_FieldDescriptor_GetOptions, + METH_NOARGS, + }, + {NULL}}; + +static PyType_Slot PyUpb_FieldDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_FieldDescriptor_Methods}, + {Py_tp_getset, PyUpb_FieldDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_FieldDescriptor_Spec = { + PYUPB_MODULE_NAME ".FieldDescriptor", + sizeof(PyUpb_DescriptorBase), + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, + PyUpb_FieldDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// FileDescriptor +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_FileDescriptor_Get(const upb_FileDef* file) { + return PyUpb_DescriptorBase_Get(kPyUpb_FileDescriptor, file, file); +} + +// These are not provided on upb_FileDef because they use the underlying +// symtab's hash table. This works for Python because everything happens under +// the GIL, but in general the caller has to guarantee that the symtab is not +// being mutated concurrently. +typedef const void* PyUpb_FileDescriptor_LookupFunc(const upb_DefPool*, + const char*); + +static const void* PyUpb_FileDescriptor_NestedLookup( + const upb_FileDef* filedef, const char* name, + PyUpb_FileDescriptor_LookupFunc* func) { + const upb_DefPool* symtab = upb_FileDef_Pool(filedef); + const char* package = upb_FileDef_Package(filedef); + if (strlen(package)) { + PyObject* qname = PyUnicode_FromFormat("%s.%s", package, name); + const void* ret = func(symtab, PyUnicode_AsUTF8AndSize(qname, NULL)); + Py_DECREF(qname); + return ret; + } else { + return func(symtab, name); + } +} + +static const void* PyUpb_FileDescriptor_LookupMessage( + const upb_FileDef* filedef, const char* name) { + return PyUpb_FileDescriptor_NestedLookup( + filedef, name, (void*)&upb_DefPool_FindMessageByName); +} + +static const void* PyUpb_FileDescriptor_LookupEnum(const upb_FileDef* filedef, + const char* name) { + return PyUpb_FileDescriptor_NestedLookup(filedef, name, + (void*)&upb_DefPool_FindEnumByName); +} + +static const void* PyUpb_FileDescriptor_LookupExtension( + const upb_FileDef* filedef, const char* name) { + return PyUpb_FileDescriptor_NestedLookup( + filedef, name, (void*)&upb_DefPool_FindExtensionByName); +} + +static const void* PyUpb_FileDescriptor_LookupService( + const upb_FileDef* filedef, const char* name) { + return PyUpb_FileDescriptor_NestedLookup( + filedef, name, (void*)&upb_DefPool_FindServiceByName); +} + +static PyObject* PyUpb_FileDescriptor_GetName(PyUpb_DescriptorBase* self, + void* closure) { + return PyUnicode_FromString(upb_FileDef_Name(self->def)); +} + +static PyObject* PyUpb_FileDescriptor_GetPool(PyObject* _self, void* closure) { + PyUpb_DescriptorBase* self = (PyUpb_DescriptorBase*)_self; + Py_INCREF(self->pool); + return self->pool; +} + +static PyObject* PyUpb_FileDescriptor_GetPackage(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (PyUpb_DescriptorBase*)_self; + return PyUnicode_FromString(upb_FileDef_Package(self->def)); +} + +static PyObject* PyUpb_FileDescriptor_GetSerializedPb(PyObject* self, + void* closure) { + return PyUpb_DescriptorBase_GetSerializedProto( + self, (PyUpb_ToProto_Func*)&upb_FileDef_ToProto, + &google_protobuf_FileDescriptorProto_msg_init); +} + +static PyObject* PyUpb_FileDescriptor_GetMessageTypesByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_FileDef_TopLevelMessageCount, + (void*)&upb_FileDef_TopLevelMessage, + (void*)&PyUpb_Descriptor_Get, + }, + (void*)&PyUpb_FileDescriptor_LookupMessage, + (void*)&upb_MessageDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetEnumTypesByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_FileDef_TopLevelEnumCount, + (void*)&upb_FileDef_TopLevelEnum, + (void*)&PyUpb_EnumDescriptor_Get, + }, + (void*)&PyUpb_FileDescriptor_LookupEnum, + (void*)&upb_EnumDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetExtensionsByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_FileDef_TopLevelExtensionCount, + (void*)&upb_FileDef_TopLevelExtension, + (void*)&PyUpb_FieldDescriptor_Get, + }, + (void*)&PyUpb_FileDescriptor_LookupExtension, + (void*)&upb_FieldDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetServicesByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_FileDef_ServiceCount, + (void*)&upb_FileDef_Service, + (void*)&PyUpb_ServiceDescriptor_Get, + }, + (void*)&PyUpb_FileDescriptor_LookupService, + (void*)&upb_ServiceDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetDependencies(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_FileDef_DependencyCount, + (void*)&upb_FileDef_Dependency, + (void*)&PyUpb_FileDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetPublicDependencies(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_FileDef_PublicDependencyCount, + (void*)&upb_FileDef_PublicDependency, + (void*)&PyUpb_FileDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_FileDescriptor_GetSyntax(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + const char* syntax = + upb_FileDef_Syntax(self->def) == kUpb_Syntax_Proto2 ? "proto2" : "proto3"; + return PyUnicode_FromString(syntax); +} + +static PyObject* PyUpb_FileDescriptor_GetHasOptions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_FileDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_FileDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_FileDef_Options(self->def), &google_protobuf_FileOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileOptions"); +} + +static PyObject* PyUpb_FileDescriptor_CopyToProto(PyObject* _self, + PyObject* py_proto) { + return PyUpb_DescriptorBase_CopyToProto( + _self, (PyUpb_ToProto_Func*)&upb_FileDef_ToProto, + &google_protobuf_FileDescriptorProto_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileDescriptorProto", py_proto); +} + +static PyGetSetDef PyUpb_FileDescriptor_Getters[] = { + {"pool", PyUpb_FileDescriptor_GetPool, NULL, "pool"}, + {"name", (getter)PyUpb_FileDescriptor_GetName, NULL, "name"}, + {"package", PyUpb_FileDescriptor_GetPackage, NULL, "package"}, + {"serialized_pb", PyUpb_FileDescriptor_GetSerializedPb}, + {"message_types_by_name", PyUpb_FileDescriptor_GetMessageTypesByName, NULL, + "Messages by name"}, + {"enum_types_by_name", PyUpb_FileDescriptor_GetEnumTypesByName, NULL, + "Enums by name"}, + {"extensions_by_name", PyUpb_FileDescriptor_GetExtensionsByName, NULL, + "Extensions by name"}, + {"services_by_name", PyUpb_FileDescriptor_GetServicesByName, NULL, + "Services by name"}, + {"dependencies", PyUpb_FileDescriptor_GetDependencies, NULL, + "Dependencies"}, + {"public_dependencies", PyUpb_FileDescriptor_GetPublicDependencies, NULL, + "Dependencies"}, + {"has_options", PyUpb_FileDescriptor_GetHasOptions, NULL, "Has Options"}, + // begin:github_only + {"syntax", PyUpb_FileDescriptor_GetSyntax, (setter)NULL, "Syntax"}, + // end:github_only + // begin:google_only +// // TODO Use this to open-source syntax deprecation. +// {"deprecated_syntax", PyUpb_FileDescriptor_GetSyntax, (setter)NULL, +// "Syntax"}, + // end:google_only + {NULL}, +}; + +static PyMethodDef PyUpb_FileDescriptor_Methods[] = { + {"GetOptions", PyUpb_FileDescriptor_GetOptions, METH_NOARGS}, + {"CopyToProto", PyUpb_FileDescriptor_CopyToProto, METH_O}, + {NULL}}; + +static PyType_Slot PyUpb_FileDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_FileDescriptor_Methods}, + {Py_tp_getset, PyUpb_FileDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_FileDescriptor_Spec = { + PYUPB_MODULE_NAME ".FileDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_FileDescriptor_Slots, +}; + +const upb_FileDef* PyUpb_FileDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_FileDescriptor); + return self ? self->def : NULL; +} + +// ----------------------------------------------------------------------------- +// MethodDescriptor +// ----------------------------------------------------------------------------- + +const upb_MethodDef* PyUpb_MethodDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_MethodDescriptor); + return self ? self->def : NULL; +} + +PyObject* PyUpb_MethodDescriptor_Get(const upb_MethodDef* m) { + const upb_FileDef* file = upb_ServiceDef_File(upb_MethodDef_Service(m)); + return PyUpb_DescriptorBase_Get(kPyUpb_MethodDescriptor, m, file); +} + +static PyObject* PyUpb_MethodDescriptor_GetName(PyObject* self, void* closure) { + const upb_MethodDef* m = PyUpb_MethodDescriptor_GetDef(self); + return PyUnicode_FromString(upb_MethodDef_Name(m)); +} + +static PyObject* PyUpb_MethodDescriptor_GetFullName(PyObject* self, + void* closure) { + const upb_MethodDef* m = PyUpb_MethodDescriptor_GetDef(self); + return PyUnicode_FromString(upb_MethodDef_FullName(m)); +} + +static PyObject* PyUpb_MethodDescriptor_GetIndex(PyObject* self, + void* closure) { + const upb_MethodDef* oneof = PyUpb_MethodDescriptor_GetDef(self); + return PyLong_FromLong(upb_MethodDef_Index(oneof)); +} + +static PyObject* PyUpb_MethodDescriptor_GetContainingService(PyObject* self, + void* closure) { + const upb_MethodDef* m = PyUpb_MethodDescriptor_GetDef(self); + return PyUpb_ServiceDescriptor_Get(upb_MethodDef_Service(m)); +} + +static PyObject* PyUpb_MethodDescriptor_GetInputType(PyObject* self, + void* closure) { + const upb_MethodDef* m = PyUpb_MethodDescriptor_GetDef(self); + return PyUpb_Descriptor_Get(upb_MethodDef_InputType(m)); +} + +static PyObject* PyUpb_MethodDescriptor_GetOutputType(PyObject* self, + void* closure) { + const upb_MethodDef* m = PyUpb_MethodDescriptor_GetDef(self); + return PyUpb_Descriptor_Get(upb_MethodDef_OutputType(m)); +} + +static PyObject* PyUpb_MethodDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_MethodDef_Options(self->def), &google_protobuf_MethodOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".MethodOptions"); +} + +static PyObject* PyUpb_MethodDescriptor_CopyToProto(PyObject* _self, + PyObject* py_proto) { + return PyUpb_DescriptorBase_CopyToProto( + _self, (PyUpb_ToProto_Func*)&upb_MethodDef_ToProto, + &google_protobuf_MethodDescriptorProto_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".MethodDescriptorProto", py_proto); +} + +static PyGetSetDef PyUpb_MethodDescriptor_Getters[] = { + {"name", PyUpb_MethodDescriptor_GetName, NULL, "Name", NULL}, + {"full_name", PyUpb_MethodDescriptor_GetFullName, NULL, "Full name", NULL}, + {"index", PyUpb_MethodDescriptor_GetIndex, NULL, "Index", NULL}, + {"containing_service", PyUpb_MethodDescriptor_GetContainingService, NULL, + "Containing service", NULL}, + {"input_type", PyUpb_MethodDescriptor_GetInputType, NULL, "Input type", + NULL}, + {"output_type", PyUpb_MethodDescriptor_GetOutputType, NULL, "Output type", + NULL}, + {NULL}}; + +static PyMethodDef PyUpb_MethodDescriptor_Methods[] = { + {"GetOptions", PyUpb_MethodDescriptor_GetOptions, METH_NOARGS}, + {"CopyToProto", PyUpb_MethodDescriptor_CopyToProto, METH_O}, + {NULL}}; + +static PyType_Slot PyUpb_MethodDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_MethodDescriptor_Methods}, + {Py_tp_getset, PyUpb_MethodDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_MethodDescriptor_Spec = { + PYUPB_MODULE_NAME ".MethodDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_MethodDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// OneofDescriptor +// ----------------------------------------------------------------------------- + +const upb_OneofDef* PyUpb_OneofDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_OneofDescriptor); + return self ? self->def : NULL; +} + +PyObject* PyUpb_OneofDescriptor_Get(const upb_OneofDef* oneof) { + const upb_FileDef* file = + upb_MessageDef_File(upb_OneofDef_ContainingType(oneof)); + return PyUpb_DescriptorBase_Get(kPyUpb_OneofDescriptor, oneof, file); +} + +static PyObject* PyUpb_OneofDescriptor_GetName(PyObject* self, void* closure) { + const upb_OneofDef* oneof = PyUpb_OneofDescriptor_GetDef(self); + return PyUnicode_FromString(upb_OneofDef_Name(oneof)); +} + +static PyObject* PyUpb_OneofDescriptor_GetFullName(PyObject* self, + void* closure) { + const upb_OneofDef* oneof = PyUpb_OneofDescriptor_GetDef(self); + return PyUnicode_FromFormat( + "%s.%s", upb_MessageDef_FullName(upb_OneofDef_ContainingType(oneof)), + upb_OneofDef_Name(oneof)); +} + +static PyObject* PyUpb_OneofDescriptor_GetIndex(PyObject* self, void* closure) { + const upb_OneofDef* oneof = PyUpb_OneofDescriptor_GetDef(self); + return PyLong_FromLong(upb_OneofDef_Index(oneof)); +} + +static PyObject* PyUpb_OneofDescriptor_GetContainingType(PyObject* self, + void* closure) { + const upb_OneofDef* oneof = PyUpb_OneofDescriptor_GetDef(self); + return PyUpb_Descriptor_Get(upb_OneofDef_ContainingType(oneof)); +} + +static PyObject* PyUpb_OneofDescriptor_GetHasOptions(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyBool_FromLong(upb_OneofDef_HasOptions(self->def)); +} + +static PyObject* PyUpb_OneofDescriptor_GetFields(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_OneofDef_FieldCount, + (void*)&upb_OneofDef_Field, + (void*)&PyUpb_FieldDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_OneofDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_OneofDef_Options(self->def), &google_protobuf_OneofOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".OneofOptions"); +} + +static PyGetSetDef PyUpb_OneofDescriptor_Getters[] = { + {"name", PyUpb_OneofDescriptor_GetName, NULL, "Name"}, + {"full_name", PyUpb_OneofDescriptor_GetFullName, NULL, "Full name"}, + {"index", PyUpb_OneofDescriptor_GetIndex, NULL, "Index"}, + {"containing_type", PyUpb_OneofDescriptor_GetContainingType, NULL, + "Containing type"}, + {"has_options", PyUpb_OneofDescriptor_GetHasOptions, NULL, "Has Options"}, + {"fields", PyUpb_OneofDescriptor_GetFields, NULL, "Fields"}, + {NULL}}; + +static PyMethodDef PyUpb_OneofDescriptor_Methods[] = { + {"GetOptions", PyUpb_OneofDescriptor_GetOptions, METH_NOARGS}, {NULL}}; + +static PyType_Slot PyUpb_OneofDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_OneofDescriptor_Methods}, + {Py_tp_getset, PyUpb_OneofDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_OneofDescriptor_Spec = { + PYUPB_MODULE_NAME ".OneofDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_OneofDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// ServiceDescriptor +// ----------------------------------------------------------------------------- + +const upb_ServiceDef* PyUpb_ServiceDescriptor_GetDef(PyObject* _self) { + PyUpb_DescriptorBase* self = + PyUpb_DescriptorBase_Check(_self, kPyUpb_ServiceDescriptor); + return self ? self->def : NULL; +} + +PyObject* PyUpb_ServiceDescriptor_Get(const upb_ServiceDef* s) { + const upb_FileDef* file = upb_ServiceDef_File(s); + return PyUpb_DescriptorBase_Get(kPyUpb_ServiceDescriptor, s, file); +} + +static PyObject* PyUpb_ServiceDescriptor_GetFullName(PyObject* self, + void* closure) { + const upb_ServiceDef* s = PyUpb_ServiceDescriptor_GetDef(self); + return PyUnicode_FromString(upb_ServiceDef_FullName(s)); +} + +static PyObject* PyUpb_ServiceDescriptor_GetName(PyObject* self, + void* closure) { + const upb_ServiceDef* s = PyUpb_ServiceDescriptor_GetDef(self); + return PyUnicode_FromString(upb_ServiceDef_Name(s)); +} + +static PyObject* PyUpb_ServiceDescriptor_GetFile(PyObject* self, + void* closure) { + const upb_ServiceDef* s = PyUpb_ServiceDescriptor_GetDef(self); + return PyUpb_FileDescriptor_Get(upb_ServiceDef_File(s)); +} + +static PyObject* PyUpb_ServiceDescriptor_GetIndex(PyObject* self, + void* closure) { + const upb_ServiceDef* s = PyUpb_ServiceDescriptor_GetDef(self); + return PyLong_FromLong(upb_ServiceDef_Index(s)); +} + +static PyObject* PyUpb_ServiceDescriptor_GetMethods(PyObject* _self, + void* closure) { + PyUpb_DescriptorBase* self = (void*)_self; + static PyUpb_GenericSequence_Funcs funcs = { + (void*)&upb_ServiceDef_MethodCount, + (void*)&upb_ServiceDef_Method, + (void*)&PyUpb_MethodDescriptor_Get, + }; + return PyUpb_GenericSequence_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_ServiceDescriptor_GetMethodsByName(PyObject* _self, + void* closure) { + static PyUpb_ByNameMap_Funcs funcs = { + { + (void*)&upb_ServiceDef_MethodCount, + (void*)&upb_ServiceDef_Method, + (void*)&PyUpb_MethodDescriptor_Get, + }, + (void*)&upb_ServiceDef_FindMethodByName, + (void*)&upb_MethodDef_Name, + }; + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_ByNameMap_New(&funcs, self->def, self->pool); +} + +static PyObject* PyUpb_ServiceDescriptor_GetOptions(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorBase* self = (void*)_self; + return PyUpb_DescriptorBase_GetOptions( + self, upb_ServiceDef_Options(self->def), &google_protobuf_ServiceOptions_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".ServiceOptions"); +} + +static PyObject* PyUpb_ServiceDescriptor_CopyToProto(PyObject* _self, + PyObject* py_proto) { + return PyUpb_DescriptorBase_CopyToProto( + _self, (PyUpb_ToProto_Func*)&upb_ServiceDef_ToProto, + &google_protobuf_ServiceDescriptorProto_msg_init, + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".ServiceDescriptorProto", py_proto); +} + +static PyObject* PyUpb_ServiceDescriptor_FindMethodByName(PyObject* _self, + PyObject* py_name) { + PyUpb_DescriptorBase* self = (void*)_self; + const char* name = PyUnicode_AsUTF8AndSize(py_name, NULL); + if (!name) return NULL; + const upb_MethodDef* method = + upb_ServiceDef_FindMethodByName(self->def, name); + if (method == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find method %.200s", name); + } + return PyUpb_MethodDescriptor_Get(method); +} + +static PyGetSetDef PyUpb_ServiceDescriptor_Getters[] = { + {"name", PyUpb_ServiceDescriptor_GetName, NULL, "Name", NULL}, + {"full_name", PyUpb_ServiceDescriptor_GetFullName, NULL, "Full name", NULL}, + {"file", PyUpb_ServiceDescriptor_GetFile, NULL, "File descriptor"}, + {"index", PyUpb_ServiceDescriptor_GetIndex, NULL, "Index", NULL}, + {"methods", PyUpb_ServiceDescriptor_GetMethods, NULL, "Methods", NULL}, + {"methods_by_name", PyUpb_ServiceDescriptor_GetMethodsByName, NULL, + "Methods by name", NULL}, + {NULL}}; + +static PyMethodDef PyUpb_ServiceDescriptor_Methods[] = { + {"GetOptions", PyUpb_ServiceDescriptor_GetOptions, METH_NOARGS}, + {"CopyToProto", PyUpb_ServiceDescriptor_CopyToProto, METH_O}, + {"FindMethodByName", PyUpb_ServiceDescriptor_FindMethodByName, METH_O}, + {NULL}}; + +static PyType_Slot PyUpb_ServiceDescriptor_Slots[] = { + DESCRIPTOR_BASE_SLOTS, + {Py_tp_methods, PyUpb_ServiceDescriptor_Methods}, + {Py_tp_getset, PyUpb_ServiceDescriptor_Getters}, + {0, NULL}}; + +static PyType_Spec PyUpb_ServiceDescriptor_Spec = { + PYUPB_MODULE_NAME ".ServiceDescriptor", // tp_name + sizeof(PyUpb_DescriptorBase), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ServiceDescriptor_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +static bool PyUpb_SetIntAttr(PyObject* obj, const char* name, int val) { + PyObject* num = PyLong_FromLong(val); + if (!num) return false; + int status = PyObject_SetAttrString(obj, name, num); + Py_DECREF(num); + return status >= 0; +} + +// These must be in the same order as PyUpb_DescriptorType in the header. +static PyType_Spec* desc_specs[] = { + &PyUpb_Descriptor_Spec, &PyUpb_EnumDescriptor_Spec, + &PyUpb_EnumValueDescriptor_Spec, &PyUpb_FieldDescriptor_Spec, + &PyUpb_FileDescriptor_Spec, &PyUpb_MethodDescriptor_Spec, + &PyUpb_OneofDescriptor_Spec, &PyUpb_ServiceDescriptor_Spec, +}; + +bool PyUpb_InitDescriptor(PyObject* m) { + PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m); + + for (size_t i = 0; i < kPyUpb_Descriptor_Count; i++) { + s->descriptor_types[i] = PyUpb_AddClass(m, desc_specs[i]); + if (!s->descriptor_types[i]) { + return false; + } + } + + PyObject* fd = (PyObject*)s->descriptor_types[kPyUpb_FieldDescriptor]; + return PyUpb_SetIntAttr(fd, "LABEL_OPTIONAL", kUpb_Label_Optional) && + PyUpb_SetIntAttr(fd, "LABEL_REPEATED", kUpb_Label_Repeated) && + PyUpb_SetIntAttr(fd, "LABEL_REQUIRED", kUpb_Label_Required) && + PyUpb_SetIntAttr(fd, "TYPE_BOOL", kUpb_FieldType_Bool) && + PyUpb_SetIntAttr(fd, "TYPE_BYTES", kUpb_FieldType_Bytes) && + PyUpb_SetIntAttr(fd, "TYPE_DOUBLE", kUpb_FieldType_Double) && + PyUpb_SetIntAttr(fd, "TYPE_ENUM", kUpb_FieldType_Enum) && + PyUpb_SetIntAttr(fd, "TYPE_FIXED32", kUpb_FieldType_Fixed32) && + PyUpb_SetIntAttr(fd, "TYPE_FIXED64", kUpb_FieldType_Fixed64) && + PyUpb_SetIntAttr(fd, "TYPE_FLOAT", kUpb_FieldType_Float) && + PyUpb_SetIntAttr(fd, "TYPE_GROUP", kUpb_FieldType_Group) && + PyUpb_SetIntAttr(fd, "TYPE_INT32", kUpb_FieldType_Int32) && + PyUpb_SetIntAttr(fd, "TYPE_INT64", kUpb_FieldType_Int64) && + PyUpb_SetIntAttr(fd, "TYPE_MESSAGE", kUpb_FieldType_Message) && + PyUpb_SetIntAttr(fd, "TYPE_SFIXED32", kUpb_FieldType_SFixed32) && + PyUpb_SetIntAttr(fd, "TYPE_SFIXED64", kUpb_FieldType_SFixed64) && + PyUpb_SetIntAttr(fd, "TYPE_SINT32", kUpb_FieldType_SInt32) && + PyUpb_SetIntAttr(fd, "TYPE_SINT64", kUpb_FieldType_SInt64) && + PyUpb_SetIntAttr(fd, "TYPE_STRING", kUpb_FieldType_String) && + PyUpb_SetIntAttr(fd, "TYPE_UINT32", kUpb_FieldType_UInt32) && + PyUpb_SetIntAttr(fd, "TYPE_UINT64", kUpb_FieldType_UInt64) && + PyUpb_SetIntAttr(fd, "CPPTYPE_INT32", CPPTYPE_INT32) && + PyUpb_SetIntAttr(fd, "CPPTYPE_INT64", CPPTYPE_INT64) && + PyUpb_SetIntAttr(fd, "CPPTYPE_UINT32", CPPTYPE_UINT32) && + PyUpb_SetIntAttr(fd, "CPPTYPE_UINT64", CPPTYPE_UINT64) && + PyUpb_SetIntAttr(fd, "CPPTYPE_DOUBLE", CPPTYPE_DOUBLE) && + PyUpb_SetIntAttr(fd, "CPPTYPE_FLOAT", CPPTYPE_FLOAT) && + PyUpb_SetIntAttr(fd, "CPPTYPE_BOOL", CPPTYPE_BOOL) && + PyUpb_SetIntAttr(fd, "CPPTYPE_ENUM", CPPTYPE_ENUM) && + PyUpb_SetIntAttr(fd, "CPPTYPE_STRING", CPPTYPE_STRING) && + PyUpb_SetIntAttr(fd, "CPPTYPE_BYTES", CPPTYPE_STRING) && + PyUpb_SetIntAttr(fd, "CPPTYPE_MESSAGE", CPPTYPE_MESSAGE); +}
diff --git a/python/descriptor.h b/python/descriptor.h new file mode 100644 index 0000000..7fa0164 --- /dev/null +++ b/python/descriptor.h
@@ -0,0 +1,85 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_DESCRIPTOR_H__ +#define PYUPB_DESCRIPTOR_H__ + +#include <stdbool.h> + +#include "python/python_api.h" +#include "upb/reflection/def.h" + +typedef enum { + kPyUpb_Descriptor = 0, + kPyUpb_EnumDescriptor = 1, + kPyUpb_EnumValueDescriptor = 2, + kPyUpb_FieldDescriptor = 3, + kPyUpb_FileDescriptor = 4, + kPyUpb_MethodDescriptor = 5, + kPyUpb_OneofDescriptor = 6, + kPyUpb_ServiceDescriptor = 7, + kPyUpb_Descriptor_Count = 8, +} PyUpb_DescriptorType; + +// Given a descriptor object |desc|, returns a Python message class object for +// the msgdef |m|, which must be from the same pool. +PyObject* PyUpb_Descriptor_GetClass(const upb_MessageDef* m); + +// Set the message descriptor's meta class. +void PyUpb_Descriptor_SetClass(PyObject* py_descriptor, PyObject* meta); + +// Returns a Python wrapper object for the given def. This will return an +// existing object if one already exists, otherwise a new object will be +// created. The caller always owns a ref on the returned object. +PyObject* PyUpb_Descriptor_Get(const upb_MessageDef* msgdef); +PyObject* PyUpb_EnumDescriptor_Get(const upb_EnumDef* enumdef); +PyObject* PyUpb_FieldDescriptor_Get(const upb_FieldDef* field); +PyObject* PyUpb_FileDescriptor_Get(const upb_FileDef* file); +PyObject* PyUpb_OneofDescriptor_Get(const upb_OneofDef* oneof); +PyObject* PyUpb_EnumValueDescriptor_Get(const upb_EnumValueDef* enumval); +PyObject* PyUpb_Descriptor_GetOrCreateWrapper(const upb_MessageDef* msg); +PyObject* PyUpb_ServiceDescriptor_Get(const upb_ServiceDef* s); +PyObject* PyUpb_MethodDescriptor_Get(const upb_MethodDef* s); + +// Returns the underlying |def| for a given wrapper object. The caller must +// have already verified that the given Python object is of the expected type. +const upb_FileDef* PyUpb_FileDescriptor_GetDef(PyObject* file); +const upb_FieldDef* PyUpb_FieldDescriptor_GetDef(PyObject* file); +const upb_MessageDef* PyUpb_Descriptor_GetDef(PyObject* _self); +const void* PyUpb_AnyDescriptor_GetDef(PyObject* _self); + +// Returns the underlying |def| for a given wrapper object. The caller must +// have already verified that the given Python object is of the expected type. +const upb_FileDef* PyUpb_FileDescriptor_GetDef(PyObject* file); + +// Module-level init. +bool PyUpb_InitDescriptor(PyObject* m); + +#endif // PYUPB_DESCRIPTOR_H__
diff --git a/python/descriptor_containers.c b/python/descriptor_containers.c new file mode 100644 index 0000000..e1eacb2 --- /dev/null +++ b/python/descriptor_containers.c
@@ -0,0 +1,816 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/descriptor_containers.h" + +#include "python/descriptor.h" +#include "python/protobuf.h" +#include "upb/reflection/def.h" + +// Implements __repr__ as str(dict(self)). +static PyObject* PyUpb_DescriptorMap_Repr(PyObject* _self) { + PyObject* dict = PyDict_New(); + PyObject* ret = NULL; + if (!dict) goto err; + if (PyDict_Merge(dict, _self, 1) != 0) goto err; + ret = PyObject_Str(dict); + +err: + Py_XDECREF(dict); + return ret; +} + +// ----------------------------------------------------------------------------- +// ByNameIterator +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + const PyUpb_ByNameMap_Funcs* funcs; + const void* parent; // upb_MessageDef*, upb_DefPool*, etc. + PyObject* parent_obj; // Python object that keeps parent alive, we own a ref. + int index; // Current iterator index. +} PyUpb_ByNameIterator; + +static PyUpb_ByNameIterator* PyUpb_ByNameIterator_Self(PyObject* obj) { + assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_name_iterator_type); + return (PyUpb_ByNameIterator*)obj; +} + +static void PyUpb_ByNameIterator_Dealloc(PyObject* _self) { + PyUpb_ByNameIterator* self = PyUpb_ByNameIterator_Self(_self); + Py_DECREF(self->parent_obj); + PyUpb_Dealloc(self); +} + +static PyObject* PyUpb_ByNameIterator_New(const PyUpb_ByNameMap_Funcs* funcs, + const void* parent, + PyObject* parent_obj) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_ByNameIterator* iter = + (void*)PyType_GenericAlloc(s->by_name_iterator_type, 0); + iter->funcs = funcs; + iter->parent = parent; + iter->parent_obj = parent_obj; + iter->index = 0; + Py_INCREF(iter->parent_obj); + return &iter->ob_base; +} + +static PyObject* PyUpb_ByNameIterator_IterNext(PyObject* _self) { + PyUpb_ByNameIterator* self = PyUpb_ByNameIterator_Self(_self); + int size = self->funcs->base.get_elem_count(self->parent); + if (self->index >= size) return NULL; + const void* elem = self->funcs->base.index(self->parent, self->index); + self->index++; + return PyUnicode_FromString(self->funcs->get_elem_name(elem)); +} + +static PyType_Slot PyUpb_ByNameIterator_Slots[] = { + {Py_tp_dealloc, PyUpb_ByNameIterator_Dealloc}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, PyUpb_ByNameIterator_IterNext}, + {0, NULL}}; + +static PyType_Spec PyUpb_ByNameIterator_Spec = { + PYUPB_MODULE_NAME "._ByNameIterator", // tp_name + sizeof(PyUpb_ByNameIterator), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ByNameIterator_Slots, +}; + +// ----------------------------------------------------------------------------- +// ByNumberIterator +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + const PyUpb_ByNumberMap_Funcs* funcs; + const void* parent; // upb_MessageDef*, upb_DefPool*, etc. + PyObject* parent_obj; // Python object that keeps parent alive, we own a ref. + int index; // Current iterator index. +} PyUpb_ByNumberIterator; + +static PyUpb_ByNumberIterator* PyUpb_ByNumberIterator_Self(PyObject* obj) { + assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_number_iterator_type); + return (PyUpb_ByNumberIterator*)obj; +} + +static void PyUpb_ByNumberIterator_Dealloc(PyObject* _self) { + PyUpb_ByNumberIterator* self = PyUpb_ByNumberIterator_Self(_self); + Py_DECREF(self->parent_obj); + PyUpb_Dealloc(self); +} + +static PyObject* PyUpb_ByNumberIterator_New( + const PyUpb_ByNumberMap_Funcs* funcs, const void* parent, + PyObject* parent_obj) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_ByNumberIterator* iter = + (void*)PyType_GenericAlloc(s->by_number_iterator_type, 0); + iter->funcs = funcs; + iter->parent = parent; + iter->parent_obj = parent_obj; + iter->index = 0; + Py_INCREF(iter->parent_obj); + return &iter->ob_base; +} + +static PyObject* PyUpb_ByNumberIterator_IterNext(PyObject* _self) { + PyUpb_ByNumberIterator* self = PyUpb_ByNumberIterator_Self(_self); + int size = self->funcs->base.get_elem_count(self->parent); + if (self->index >= size) return NULL; + const void* elem = self->funcs->base.index(self->parent, self->index); + self->index++; + return PyLong_FromLong(self->funcs->get_elem_num(elem)); +} + +static PyType_Slot PyUpb_ByNumberIterator_Slots[] = { + {Py_tp_dealloc, PyUpb_ByNumberIterator_Dealloc}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, PyUpb_ByNumberIterator_IterNext}, + {0, NULL}}; + +static PyType_Spec PyUpb_ByNumberIterator_Spec = { + PYUPB_MODULE_NAME "._ByNumberIterator", // tp_name + sizeof(PyUpb_ByNumberIterator), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ByNumberIterator_Slots, +}; + +// ----------------------------------------------------------------------------- +// GenericSequence +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + const PyUpb_GenericSequence_Funcs* funcs; + const void* parent; // upb_MessageDef*, upb_DefPool*, etc. + PyObject* parent_obj; // Python object that keeps parent alive, we own a ref. +} PyUpb_GenericSequence; + +PyUpb_GenericSequence* PyUpb_GenericSequence_Self(PyObject* obj) { + assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->generic_sequence_type); + return (PyUpb_GenericSequence*)obj; +} + +static void PyUpb_GenericSequence_Dealloc(PyObject* _self) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + Py_CLEAR(self->parent_obj); + PyUpb_Dealloc(self); +} + +PyObject* PyUpb_GenericSequence_New(const PyUpb_GenericSequence_Funcs* funcs, + const void* parent, PyObject* parent_obj) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_GenericSequence* seq = + (PyUpb_GenericSequence*)PyType_GenericAlloc(s->generic_sequence_type, 0); + seq->funcs = funcs; + seq->parent = parent; + seq->parent_obj = parent_obj; + Py_INCREF(parent_obj); + return &seq->ob_base; +} + +static Py_ssize_t PyUpb_GenericSequence_Length(PyObject* _self) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + return self->funcs->get_elem_count(self->parent); +} + +static PyObject* PyUpb_GenericSequence_GetItem(PyObject* _self, + Py_ssize_t index) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + Py_ssize_t size = self->funcs->get_elem_count(self->parent); + if (index < 0) { + index += size; + } + if (index < 0 || index >= size) { + PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index); + return NULL; + } + const void* elem = self->funcs->index(self->parent, index); + return self->funcs->get_elem_wrapper(elem); +} + +// A sequence container can only be equal to another sequence container, or (for +// backward compatibility) to a list containing the same items. +// Returns 1 if equal, 0 if unequal, -1 on error. +static int PyUpb_GenericSequence_IsEqual(PyUpb_GenericSequence* self, + PyObject* other) { + // Check the identity of C++ pointers. + if (PyObject_TypeCheck(other, Py_TYPE(self))) { + PyUpb_GenericSequence* other_seq = (void*)other; + return self->parent == other_seq->parent && self->funcs == other_seq->funcs; + } + + if (!PyList_Check(other)) return 0; + + // return list(self) == other + // We can clamp `i` to int because GenericSequence uses int for size (this + // is useful when we do int iteration below). + int n = PyUpb_GenericSequence_Length((PyObject*)self); + if ((Py_ssize_t)n != PyList_Size(other)) { + return false; + } + + PyObject* item1; + for (int i = 0; i < n; i++) { + item1 = PyUpb_GenericSequence_GetItem((PyObject*)self, i); + PyObject* item2 = PyList_GetItem(other, i); + if (!item1 || !item2) goto error; + int cmp = PyObject_RichCompareBool(item1, item2, Py_EQ); + Py_DECREF(item1); + if (cmp != 1) return cmp; + } + // All items were found and equal + return 1; + +error: + Py_XDECREF(item1); + return -1; +} + +static PyObject* PyUpb_GenericSequence_RichCompare(PyObject* _self, + PyObject* other, int opid) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + if (opid != Py_EQ && opid != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + bool ret = PyUpb_GenericSequence_IsEqual(self, other); + if (opid == Py_NE) ret = !ret; + return PyBool_FromLong(ret); +} + +static PyObject* PyUpb_GenericSequence_Subscript(PyObject* _self, + PyObject* item) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + Py_ssize_t size = self->funcs->get_elem_count(self->parent); + Py_ssize_t idx, count, step; + if (!PyUpb_IndexToRange(item, size, &idx, &count, &step)) return NULL; + if (step == 0) { + return PyUpb_GenericSequence_GetItem(_self, idx); + } else { + PyObject* list = PyList_New(count); + for (Py_ssize_t i = 0; i < count; i++, idx += step) { + const void* elem = self->funcs->index(self->parent, idx); + PyList_SetItem(list, i, self->funcs->get_elem_wrapper(elem)); + } + return list; + } +} + +// Linear search. Could optimize this in some cases (defs that have index), +// but not all (FileDescriptor.dependencies). +static int PyUpb_GenericSequence_Find(PyObject* _self, PyObject* item) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + const void* item_ptr = PyUpb_AnyDescriptor_GetDef(item); + int count = self->funcs->get_elem_count(self->parent); + for (int i = 0; i < count; i++) { + if (self->funcs->index(self->parent, i) == item_ptr) { + return i; + } + } + return -1; +} + +static PyObject* PyUpb_GenericSequence_Index(PyObject* self, PyObject* item) { + int position = PyUpb_GenericSequence_Find(self, item); + if (position < 0) { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } else { + return PyLong_FromLong(position); + } +} + +static PyObject* PyUpb_GenericSequence_Count(PyObject* _self, PyObject* item) { + PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self); + const void* item_ptr = PyUpb_AnyDescriptor_GetDef(item); + int n = self->funcs->get_elem_count(self->parent); + int count = 0; + for (int i = 0; i < n; i++) { + if (self->funcs->index(self->parent, i) == item_ptr) { + count++; + } + } + return PyLong_FromLong(count); +} + +static PyObject* PyUpb_GenericSequence_Append(PyObject* self, PyObject* args) { + PyErr_Format(PyExc_TypeError, "'%R' is not a mutable sequence", self); + return NULL; +} + +static PyMethodDef PyUpb_GenericSequence_Methods[] = { + {"index", PyUpb_GenericSequence_Index, METH_O}, + {"count", PyUpb_GenericSequence_Count, METH_O}, + {"append", PyUpb_GenericSequence_Append, METH_O}, + // This was implemented for Python/C++ but so far has not been required. + //{ "__reversed__", (PyCFunction)Reversed, METH_NOARGS, }, + {NULL}}; + +static PyType_Slot PyUpb_GenericSequence_Slots[] = { + {Py_tp_dealloc, &PyUpb_GenericSequence_Dealloc}, + {Py_tp_methods, &PyUpb_GenericSequence_Methods}, + {Py_sq_length, PyUpb_GenericSequence_Length}, + {Py_sq_item, PyUpb_GenericSequence_GetItem}, + {Py_tp_richcompare, &PyUpb_GenericSequence_RichCompare}, + {Py_mp_subscript, PyUpb_GenericSequence_Subscript}, + // These were implemented for Python/C++ but so far have not been required. + // {Py_tp_repr, &PyUpb_GenericSequence_Repr}, + // {Py_sq_contains, PyUpb_GenericSequence_Contains}, + // {Py_mp_ass_subscript, PyUpb_GenericSequence_AssignSubscript}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_GenericSequence_Spec = { + PYUPB_MODULE_NAME "._GenericSequence", // tp_name + sizeof(PyUpb_GenericSequence), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_GenericSequence_Slots, +}; + +// ----------------------------------------------------------------------------- +// ByNameMap +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + const PyUpb_ByNameMap_Funcs* funcs; + const void* parent; // upb_MessageDef*, upb_DefPool*, etc. + PyObject* parent_obj; // Python object that keeps parent alive, we own a ref. +} PyUpb_ByNameMap; + +PyUpb_ByNameMap* PyUpb_ByNameMap_Self(PyObject* obj) { + assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_name_map_type); + return (PyUpb_ByNameMap*)obj; +} + +static void PyUpb_ByNameMap_Dealloc(PyObject* _self) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + Py_DECREF(self->parent_obj); + PyUpb_Dealloc(self); +} + +PyObject* PyUpb_ByNameMap_New(const PyUpb_ByNameMap_Funcs* funcs, + const void* parent, PyObject* parent_obj) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_ByNameMap* map = (void*)PyType_GenericAlloc(s->by_name_map_type, 0); + map->funcs = funcs; + map->parent = parent; + map->parent_obj = parent_obj; + Py_INCREF(parent_obj); + return &map->ob_base; +} + +static Py_ssize_t PyUpb_ByNameMap_Length(PyObject* _self) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + return self->funcs->base.get_elem_count(self->parent); +} + +static PyObject* PyUpb_ByNameMap_Subscript(PyObject* _self, PyObject* key) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + const char* name = PyUpb_GetStrData(key); + const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL; + + if (!name && PyObject_Hash(key) == -1) return NULL; + + if (elem) { + return self->funcs->base.get_elem_wrapper(elem); + } else { + PyErr_SetObject(PyExc_KeyError, key); + return NULL; + } +} + +static int PyUpb_ByNameMap_AssignSubscript(PyObject* self, PyObject* key, + PyObject* value) { + PyErr_Format(PyExc_TypeError, PYUPB_MODULE_NAME + ".ByNameMap' object does not support item assignment"); + return -1; +} + +static int PyUpb_ByNameMap_Contains(PyObject* _self, PyObject* key) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + const char* name = PyUpb_GetStrData(key); + const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL; + if (!name && PyObject_Hash(key) == -1) return -1; + return elem ? 1 : 0; +} + +static PyObject* PyUpb_ByNameMap_Get(PyObject* _self, PyObject* args) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + PyObject* key; + PyObject* default_value = Py_None; + if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &default_value)) { + return NULL; + } + + const char* name = PyUpb_GetStrData(key); + const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL; + + if (!name && PyObject_Hash(key) == -1) return NULL; + + if (elem) { + return self->funcs->base.get_elem_wrapper(elem); + } else { + Py_INCREF(default_value); + return default_value; + } +} + +static PyObject* PyUpb_ByNameMap_GetIter(PyObject* _self) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + return PyUpb_ByNameIterator_New(self->funcs, self->parent, self->parent_obj); +} + +static PyObject* PyUpb_ByNameMap_Keys(PyObject* _self, PyObject* args) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + PyObject* key = PyUnicode_FromString(self->funcs->get_elem_name(elem)); + if (!key) goto error; + PyList_SetItem(ret, i, key); + } + return ret; + +error: + Py_XDECREF(ret); + return NULL; +} + +static PyObject* PyUpb_ByNameMap_Values(PyObject* _self, PyObject* args) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + PyObject* py_elem = self->funcs->base.get_elem_wrapper(elem); + if (!py_elem) goto error; + PyList_SetItem(ret, i, py_elem); + } + return ret; + +error: + Py_XDECREF(ret); + return NULL; +} + +static PyObject* PyUpb_ByNameMap_Items(PyObject* _self, PyObject* args) { + PyUpb_ByNameMap* self = (PyUpb_ByNameMap*)_self; + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + PyObject* item; + PyObject* py_elem; + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + item = PyTuple_New(2); + py_elem = self->funcs->base.get_elem_wrapper(elem); + if (!item || !py_elem) goto error; + PyTuple_SetItem(item, 0, + PyUnicode_FromString(self->funcs->get_elem_name(elem))); + PyTuple_SetItem(item, 1, py_elem); + PyList_SetItem(ret, i, item); + } + return ret; + +error: + Py_XDECREF(py_elem); + Py_XDECREF(item); + Py_XDECREF(ret); + return NULL; +} + +// A mapping container can only be equal to another mapping container, or (for +// backward compatibility) to a dict containing the same items. +// Returns 1 if equal, 0 if unequal, -1 on error. +static int PyUpb_ByNameMap_IsEqual(PyUpb_ByNameMap* self, PyObject* other) { + // Check the identity of C++ pointers. + if (PyObject_TypeCheck(other, Py_TYPE(self))) { + PyUpb_ByNameMap* other_map = (void*)other; + return self->parent == other_map->parent && self->funcs == other_map->funcs; + } + + if (!PyDict_Check(other)) return 0; + + PyObject* self_dict = PyDict_New(); + PyDict_Merge(self_dict, (PyObject*)self, 0); + int eq = PyObject_RichCompareBool(self_dict, other, Py_EQ); + Py_DECREF(self_dict); + return eq; +} + +static PyObject* PyUpb_ByNameMap_RichCompare(PyObject* _self, PyObject* other, + int opid) { + PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self); + if (opid != Py_EQ && opid != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + bool ret = PyUpb_ByNameMap_IsEqual(self, other); + if (opid == Py_NE) ret = !ret; + return PyBool_FromLong(ret); +} + +static PyMethodDef PyUpb_ByNameMap_Methods[] = { + {"get", (PyCFunction)&PyUpb_ByNameMap_Get, METH_VARARGS}, + {"keys", PyUpb_ByNameMap_Keys, METH_NOARGS}, + {"values", PyUpb_ByNameMap_Values, METH_NOARGS}, + {"items", PyUpb_ByNameMap_Items, METH_NOARGS}, + {NULL}}; + +static PyType_Slot PyUpb_ByNameMap_Slots[] = { + {Py_mp_ass_subscript, PyUpb_ByNameMap_AssignSubscript}, + {Py_mp_length, PyUpb_ByNameMap_Length}, + {Py_mp_subscript, PyUpb_ByNameMap_Subscript}, + {Py_sq_contains, &PyUpb_ByNameMap_Contains}, + {Py_tp_dealloc, &PyUpb_ByNameMap_Dealloc}, + {Py_tp_iter, PyUpb_ByNameMap_GetIter}, + {Py_tp_methods, &PyUpb_ByNameMap_Methods}, + {Py_tp_repr, &PyUpb_DescriptorMap_Repr}, + {Py_tp_richcompare, &PyUpb_ByNameMap_RichCompare}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_ByNameMap_Spec = { + PYUPB_MODULE_NAME "._ByNameMap", // tp_name + sizeof(PyUpb_ByNameMap), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ByNameMap_Slots, +}; + +// ----------------------------------------------------------------------------- +// ByNumberMap +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + const PyUpb_ByNumberMap_Funcs* funcs; + const void* parent; // upb_MessageDef*, upb_DefPool*, etc. + PyObject* parent_obj; // Python object that keeps parent alive, we own a ref. +} PyUpb_ByNumberMap; + +PyUpb_ByNumberMap* PyUpb_ByNumberMap_Self(PyObject* obj) { + assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_number_map_type); + return (PyUpb_ByNumberMap*)obj; +} + +PyObject* PyUpb_ByNumberMap_New(const PyUpb_ByNumberMap_Funcs* funcs, + const void* parent, PyObject* parent_obj) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_ByNumberMap* map = (void*)PyType_GenericAlloc(s->by_number_map_type, 0); + map->funcs = funcs; + map->parent = parent; + map->parent_obj = parent_obj; + Py_INCREF(parent_obj); + return &map->ob_base; +} + +static void PyUpb_ByNumberMap_Dealloc(PyObject* _self) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + Py_DECREF(self->parent_obj); + PyUpb_Dealloc(self); +} + +static Py_ssize_t PyUpb_ByNumberMap_Length(PyObject* _self) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + return self->funcs->base.get_elem_count(self->parent); +} + +static const void* PyUpb_ByNumberMap_LookupHelper(PyUpb_ByNumberMap* self, + PyObject* key) { + long num = PyLong_AsLong(key); + if (num == -1 && PyErr_Occurred()) { + PyErr_Clear(); + // Ensure that the key is hashable (this will raise an error if not). + PyObject_Hash(key); + return NULL; + } else { + return self->funcs->lookup(self->parent, num); + } +} + +static PyObject* PyUpb_ByNumberMap_Subscript(PyObject* _self, PyObject* key) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key); + if (elem) { + return self->funcs->base.get_elem_wrapper(elem); + } else { + if (!PyErr_Occurred()) { + PyErr_SetObject(PyExc_KeyError, key); + } + return NULL; + } +} + +static int PyUpb_ByNumberMap_AssignSubscript(PyObject* self, PyObject* key, + PyObject* value) { + PyErr_Format(PyExc_TypeError, PYUPB_MODULE_NAME + ".ByNumberMap' object does not support item assignment"); + return -1; +} + +static PyObject* PyUpb_ByNumberMap_Get(PyObject* _self, PyObject* args) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + PyObject* key; + PyObject* default_value = Py_None; + if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &default_value)) { + return NULL; + } + + const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key); + if (elem) { + return self->funcs->base.get_elem_wrapper(elem); + } else if (PyErr_Occurred()) { + return NULL; + } else { + return PyUpb_NewRef(default_value); + } +} + +static PyObject* PyUpb_ByNumberMap_GetIter(PyObject* _self) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + return PyUpb_ByNumberIterator_New(self->funcs, self->parent, + self->parent_obj); +} + +static PyObject* PyUpb_ByNumberMap_Keys(PyObject* _self, PyObject* args) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + PyObject* key = PyLong_FromLong(self->funcs->get_elem_num(elem)); + if (!key) goto error; + PyList_SetItem(ret, i, key); + } + return ret; + +error: + Py_XDECREF(ret); + return NULL; +} + +static PyObject* PyUpb_ByNumberMap_Values(PyObject* _self, PyObject* args) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + PyObject* py_elem = self->funcs->base.get_elem_wrapper(elem); + if (!py_elem) goto error; + PyList_SetItem(ret, i, py_elem); + } + return ret; + +error: + Py_XDECREF(ret); + return NULL; +} + +static PyObject* PyUpb_ByNumberMap_Items(PyObject* _self, PyObject* args) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + int n = self->funcs->base.get_elem_count(self->parent); + PyObject* ret = PyList_New(n); + PyObject* item; + PyObject* py_elem; + if (!ret) return NULL; + for (int i = 0; i < n; i++) { + const void* elem = self->funcs->base.index(self->parent, i); + int number = self->funcs->get_elem_num(elem); + item = PyTuple_New(2); + py_elem = self->funcs->base.get_elem_wrapper(elem); + if (!item || !py_elem) goto error; + PyTuple_SetItem(item, 0, PyLong_FromLong(number)); + PyTuple_SetItem(item, 1, py_elem); + PyList_SetItem(ret, i, item); + } + return ret; + +error: + Py_XDECREF(py_elem); + Py_XDECREF(item); + Py_XDECREF(ret); + return NULL; +} + +static int PyUpb_ByNumberMap_Contains(PyObject* _self, PyObject* key) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key); + if (elem) return 1; + if (PyErr_Occurred()) return -1; + return 0; +} + +// A mapping container can only be equal to another mapping container, or (for +// backward compatibility) to a dict containing the same items. +// Returns 1 if equal, 0 if unequal, -1 on error. +static int PyUpb_ByNumberMap_IsEqual(PyUpb_ByNumberMap* self, PyObject* other) { + // Check the identity of C++ pointers. + if (PyObject_TypeCheck(other, Py_TYPE(self))) { + PyUpb_ByNumberMap* other_map = (void*)other; + return self->parent == other_map->parent && self->funcs == other_map->funcs; + } + + if (!PyDict_Check(other)) return 0; + + PyObject* self_dict = PyDict_New(); + PyDict_Merge(self_dict, (PyObject*)self, 0); + int eq = PyObject_RichCompareBool(self_dict, other, Py_EQ); + Py_DECREF(self_dict); + return eq; +} + +static PyObject* PyUpb_ByNumberMap_RichCompare(PyObject* _self, PyObject* other, + int opid) { + PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self); + if (opid != Py_EQ && opid != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + bool ret = PyUpb_ByNumberMap_IsEqual(self, other); + if (opid == Py_NE) ret = !ret; + return PyBool_FromLong(ret); +} + +static PyMethodDef PyUpb_ByNumberMap_Methods[] = { + {"get", (PyCFunction)&PyUpb_ByNumberMap_Get, METH_VARARGS}, + {"keys", PyUpb_ByNumberMap_Keys, METH_NOARGS}, + {"values", PyUpb_ByNumberMap_Values, METH_NOARGS}, + {"items", PyUpb_ByNumberMap_Items, METH_NOARGS}, + {NULL}}; + +static PyType_Slot PyUpb_ByNumberMap_Slots[] = { + {Py_mp_ass_subscript, PyUpb_ByNumberMap_AssignSubscript}, + {Py_mp_length, PyUpb_ByNumberMap_Length}, + {Py_mp_subscript, PyUpb_ByNumberMap_Subscript}, + {Py_sq_contains, &PyUpb_ByNumberMap_Contains}, + {Py_tp_dealloc, &PyUpb_ByNumberMap_Dealloc}, + {Py_tp_iter, PyUpb_ByNumberMap_GetIter}, + {Py_tp_methods, &PyUpb_ByNumberMap_Methods}, + {Py_tp_repr, &PyUpb_DescriptorMap_Repr}, + {Py_tp_richcompare, &PyUpb_ByNumberMap_RichCompare}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_ByNumberMap_Spec = { + PYUPB_MODULE_NAME "._ByNumberMap", // tp_name + sizeof(PyUpb_ByNumberMap), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ByNumberMap_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +bool PyUpb_InitDescriptorContainers(PyObject* m) { + PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m); + + s->by_name_map_type = PyUpb_AddClass(m, &PyUpb_ByNameMap_Spec); + s->by_number_map_type = PyUpb_AddClass(m, &PyUpb_ByNumberMap_Spec); + s->by_name_iterator_type = PyUpb_AddClass(m, &PyUpb_ByNameIterator_Spec); + s->by_number_iterator_type = PyUpb_AddClass(m, &PyUpb_ByNumberIterator_Spec); + s->generic_sequence_type = PyUpb_AddClass(m, &PyUpb_GenericSequence_Spec); + + return s->by_name_map_type && s->by_number_map_type && + s->by_name_iterator_type && s->by_number_iterator_type && + s->generic_sequence_type; +}
diff --git a/python/descriptor_containers.h b/python/descriptor_containers.h new file mode 100644 index 0000000..5b2b1fa --- /dev/null +++ b/python/descriptor_containers.h
@@ -0,0 +1,117 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_DESCRIPTOR_CONTAINERS_H__ +#define PYUPB_DESCRIPTOR_CONTAINERS_H__ + +// This file defines immutable Python containiner types whose data comes from +// an underlying descriptor (def). +// +// Because there are many instances of these types that vend different kinds of +// data (fields, oneofs, enums, etc) these types accept a "vtable" of function +// pointers. This saves us from having to define numerous distinct Python types +// for each kind of data we want to vend. +// +// The underlying upb APIs follow a consistent pattern that allows us to use +// those functions directly inside these vtables, greatly reducing the amount of +// "adaptor" code we need to write. + +#include <stdbool.h> + +#include "protobuf.h" +#include "upb/reflection/def.h" + +// ----------------------------------------------------------------------------- +// PyUpb_GenericSequence +// ----------------------------------------------------------------------------- + +// A Python object that vends a sequence of descriptors. + +typedef struct { + // Returns the number of elements in the map. + int (*get_elem_count)(const void* parent); + // Returns an element by index. + const void* (*index)(const void* parent, int idx); + // Returns a Python object wrapping this element, caller owns a ref. + PyObject* (*get_elem_wrapper)(const void* elem); +} PyUpb_GenericSequence_Funcs; + +// Returns a new GenericSequence. The vtable `funcs` must outlive this object +// (generally it should be static). The GenericSequence will take a ref on +// `parent_obj`, which must be sufficient to keep `parent` alive. The object +// `parent` will be passed as an argument to the functions in `funcs`. +PyObject* PyUpb_GenericSequence_New(const PyUpb_GenericSequence_Funcs* funcs, + const void* parent, PyObject* parent_obj); + +// ----------------------------------------------------------------------------- +// PyUpb_ByNameMap +// ----------------------------------------------------------------------------- + +// A Python object that vends a name->descriptor map. + +typedef struct { + PyUpb_GenericSequence_Funcs base; + // Looks up by name and returns either a pointer to the element or NULL. + const void* (*lookup)(const void* parent, const char* key); + // Returns the name associated with this element. + const char* (*get_elem_name)(const void* elem); +} PyUpb_ByNameMap_Funcs; + +// Returns a new ByNameMap. The vtable `funcs` must outlive this object +// (generally it should be static). The ByNameMap will take a ref on +// `parent_obj`, which must be sufficient to keep `parent` alive. The object +// `parent` will be passed as an argument to the functions in `funcs`. +PyObject* PyUpb_ByNameMap_New(const PyUpb_ByNameMap_Funcs* funcs, + const void* parent, PyObject* parent_obj); + +// ----------------------------------------------------------------------------- +// PyUpb_ByNumberMap +// ----------------------------------------------------------------------------- + +// A Python object that vends a number->descriptor map. + +typedef struct { + PyUpb_GenericSequence_Funcs base; + // Looks up by name and returns either a pointer to the element or NULL. + const void* (*lookup)(const void* parent, int num); + // Returns the name associated with this element. + int (*get_elem_num)(const void* elem); +} PyUpb_ByNumberMap_Funcs; + +// Returns a new ByNumberMap. The vtable `funcs` must outlive this object +// (generally it should be static). The ByNumberMap will take a ref on +// `parent_obj`, which must be sufficient to keep `parent` alive. The object +// `parent` will be passed as an argument to the functions in `funcs`. +PyObject* PyUpb_ByNumberMap_New(const PyUpb_ByNumberMap_Funcs* funcs, + const void* parent, PyObject* parent_obj); + +bool PyUpb_InitDescriptorContainers(PyObject* m); + +#endif // PYUPB_DESCRIPTOR_CONTAINERS_H__
diff --git a/python/descriptor_pool.c b/python/descriptor_pool.c new file mode 100644 index 0000000..ee41677 --- /dev/null +++ b/python/descriptor_pool.c
@@ -0,0 +1,652 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/descriptor_pool.h" + +#include "google/protobuf/descriptor.upbdefs.h" +#include "python/convert.h" +#include "python/descriptor.h" +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/reflection/def.h" +#include "upb/util/def_to_proto.h" + +// ----------------------------------------------------------------------------- +// DescriptorPool +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + upb_DefPool* symtab; + PyObject* db; // The DescriptorDatabase underlying this pool. May be NULL. +} PyUpb_DescriptorPool; + +PyObject* PyUpb_DescriptorPool_GetDefaultPool(void) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + return s->default_pool; +} + +const upb_MessageDef* PyUpb_DescriptorPool_GetFileProtoDef(void) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + if (!s->c_descriptor_symtab) { + s->c_descriptor_symtab = upb_DefPool_New(); + } + return google_protobuf_FileDescriptorProto_getmsgdef(s->c_descriptor_symtab); +} + +static PyObject* PyUpb_DescriptorPool_DoCreateWithCache( + PyTypeObject* type, PyObject* db, PyUpb_WeakMap* obj_cache) { + PyUpb_DescriptorPool* pool = (void*)PyType_GenericAlloc(type, 0); + pool->symtab = upb_DefPool_New(); + pool->db = db; + Py_XINCREF(pool->db); + PyUpb_WeakMap_Add(obj_cache, pool->symtab, &pool->ob_base); + return &pool->ob_base; +} + +static PyObject* PyUpb_DescriptorPool_DoCreate(PyTypeObject* type, + PyObject* db) { + return PyUpb_DescriptorPool_DoCreateWithCache(type, db, + PyUpb_ObjCache_Instance()); +} + +upb_DefPool* PyUpb_DescriptorPool_GetSymtab(PyObject* pool) { + return ((PyUpb_DescriptorPool*)pool)->symtab; +} + +static int PyUpb_DescriptorPool_Traverse(PyUpb_DescriptorPool* self, + visitproc visit, void* arg) { + Py_VISIT(self->db); + return 0; +} + +static int PyUpb_DescriptorPool_Clear(PyUpb_DescriptorPool* self) { + Py_CLEAR(self->db); + return 0; +} + +PyObject* PyUpb_DescriptorPool_Get(const upb_DefPool* symtab) { + PyObject* pool = PyUpb_ObjCache_Get(symtab); + assert(pool); + return pool; +} + +static void PyUpb_DescriptorPool_Dealloc(PyUpb_DescriptorPool* self) { + PyUpb_DescriptorPool_Clear(self); + upb_DefPool_Free(self->symtab); + PyUpb_ObjCache_Delete(self->symtab); + PyUpb_Dealloc(self); +} + +/* + * DescriptorPool.__new__() + * + * Implements: + * DescriptorPool(descriptor_db=None) + */ +static PyObject* PyUpb_DescriptorPool_New(PyTypeObject* type, PyObject* args, + PyObject* kwargs) { + char* kwlist[] = {"descriptor_db", 0}; + PyObject* db = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &db)) { + return NULL; + } + + if (db == Py_None) db = NULL; + return PyUpb_DescriptorPool_DoCreate(type, db); +} + +static PyObject* PyUpb_DescriptorPool_DoAdd(PyObject* _self, + PyObject* file_desc); + +static bool PyUpb_DescriptorPool_TryLoadFileProto(PyUpb_DescriptorPool* self, + PyObject* proto) { + if (proto == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + // Expected error: item was simply not found. + PyErr_Clear(); + return true; // We didn't accomplish our goal, but we didn't error out. + } + return false; + } + if (proto == Py_None) return true; + PyObject* ret = PyUpb_DescriptorPool_DoAdd((PyObject*)self, proto); + bool ok = ret != NULL; + Py_XDECREF(ret); + return ok; +} + +static bool PyUpb_DescriptorPool_TryLoadSymbol(PyUpb_DescriptorPool* self, + PyObject* sym) { + if (!self->db) return false; + PyObject* file_proto = + PyObject_CallMethod(self->db, "FindFileContainingSymbol", "O", sym); + bool ret = PyUpb_DescriptorPool_TryLoadFileProto(self, file_proto); + Py_XDECREF(file_proto); + return ret; +} + +static bool PyUpb_DescriptorPool_TryLoadFilename(PyUpb_DescriptorPool* self, + PyObject* filename) { + if (!self->db) return false; + PyObject* file_proto = + PyObject_CallMethod(self->db, "FindFileByName", "O", filename); + bool ret = PyUpb_DescriptorPool_TryLoadFileProto(self, file_proto); + Py_XDECREF(file_proto); + return ret; +} + +bool PyUpb_DescriptorPool_CheckNoDatabase(PyObject* _self) { return true; } + +static bool PyUpb_DescriptorPool_LoadDependentFiles( + PyUpb_DescriptorPool* self, google_protobuf_FileDescriptorProto* proto) { + size_t n; + const upb_StringView* deps = + google_protobuf_FileDescriptorProto_dependency(proto, &n); + for (size_t i = 0; i < n; i++) { + const upb_FileDef* dep = upb_DefPool_FindFileByNameWithSize( + self->symtab, deps[i].data, deps[i].size); + if (!dep) { + PyObject* filename = + PyUnicode_FromStringAndSize(deps[i].data, deps[i].size); + if (!filename) return false; + bool ok = PyUpb_DescriptorPool_TryLoadFilename(self, filename); + Py_DECREF(filename); + if (!ok) return false; + } + } + return true; +} + +static PyObject* PyUpb_DescriptorPool_DoAddSerializedFile( + PyObject* _self, PyObject* serialized_pb) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + upb_Arena* arena = upb_Arena_New(); + if (!arena) PYUPB_RETURN_OOM; + PyObject* result = NULL; + + char* buf; + Py_ssize_t size; + if (PyBytes_AsStringAndSize(serialized_pb, &buf, &size) < 0) { + goto done; + } + + google_protobuf_FileDescriptorProto* proto = + google_protobuf_FileDescriptorProto_parse(buf, size, arena); + if (!proto) { + PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!"); + goto done; + } + + upb_StringView name = google_protobuf_FileDescriptorProto_name(proto); + const upb_FileDef* file = + upb_DefPool_FindFileByNameWithSize(self->symtab, name.data, name.size); + + if (file) { + // If the existing file is equal to the new file, then silently ignore the + // duplicate add. + google_protobuf_FileDescriptorProto* existing = + upb_FileDef_ToProto(file, arena); + if (!existing) { + PyErr_SetNone(PyExc_MemoryError); + goto done; + } + const upb_MessageDef* m = PyUpb_DescriptorPool_GetFileProtoDef(); + if (upb_Message_IsEqual(proto, existing, m)) { + result = PyUpb_FileDescriptor_Get(file); + goto done; + } + } + + if (self->db) { + if (!PyUpb_DescriptorPool_LoadDependentFiles(self, proto)) goto done; + } + + upb_Status status; + upb_Status_Clear(&status); + + const upb_FileDef* filedef = + upb_DefPool_AddFile(self->symtab, proto, &status); + if (!filedef) { + PyErr_Format(PyExc_TypeError, + "Couldn't build proto file into descriptor pool: %s", + upb_Status_ErrorMessage(&status)); + goto done; + } + + result = PyUpb_FileDescriptor_Get(filedef); + +done: + upb_Arena_Free(arena); + return result; +} + +static PyObject* PyUpb_DescriptorPool_DoAdd(PyObject* _self, + PyObject* file_desc) { + if (!PyUpb_Message_Verify(file_desc)) return NULL; + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(file_desc); + const char* file_proto_name = + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileDescriptorProto"; + if (strcmp(upb_MessageDef_FullName(m), file_proto_name) != 0) { + return PyErr_Format(PyExc_TypeError, "Can only add FileDescriptorProto"); + } + PyObject* subargs = PyTuple_New(0); + if (!subargs) return NULL; + PyObject* serialized = + PyUpb_Message_SerializeToString(file_desc, subargs, NULL); + Py_DECREF(subargs); + if (!serialized) return NULL; + PyObject* ret = PyUpb_DescriptorPool_DoAddSerializedFile(_self, serialized); + Py_DECREF(serialized); + return ret; +} + +/* + * PyUpb_DescriptorPool_AddSerializedFile() + * + * Implements: + * DescriptorPool.AddSerializedFile(self, serialized_file_descriptor) + * + * Adds the given serialized FileDescriptorProto to the pool. + */ +static PyObject* PyUpb_DescriptorPool_AddSerializedFile( + PyObject* _self, PyObject* serialized_pb) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + if (self->db) { + PyErr_SetString( + PyExc_ValueError, + "Cannot call AddSerializedFile on a DescriptorPool that uses a " + "DescriptorDatabase. Add your file to the underlying database."); + return false; + } + return PyUpb_DescriptorPool_DoAddSerializedFile(_self, serialized_pb); +} + +static PyObject* PyUpb_DescriptorPool_Add(PyObject* _self, + PyObject* file_desc) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + if (self->db) { + PyErr_SetString( + PyExc_ValueError, + "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. " + "Add your file to the underlying database."); + return false; + } + return PyUpb_DescriptorPool_DoAdd(_self, file_desc); +} + +/* + * PyUpb_DescriptorPool_FindFileByName() + * + * Implements: + * DescriptorPool.FindFileByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindFileByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_FileDef* file = upb_DefPool_FindFileByName(self->symtab, name); + if (file == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadFilename(self, arg)) return NULL; + file = upb_DefPool_FindFileByName(self->symtab, name); + } + if (file == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s", name); + } + + return PyUpb_FileDescriptor_Get(file); +} + +/* + * PyUpb_DescriptorPool_FindExtensionByName() + * + * Implements: + * DescriptorPool.FindExtensionByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindExtensionByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_FieldDef* field = + upb_DefPool_FindExtensionByName(self->symtab, name); + if (field == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + field = upb_DefPool_FindExtensionByName(self->symtab, name); + } + if (field == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find extension %.200s", name); + } + + return PyUpb_FieldDescriptor_Get(field); +} + +/* + * PyUpb_DescriptorPool_FindMessageTypeByName() + * + * Implements: + * DescriptorPool.FindMessageTypeByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindMessageTypeByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_MessageDef* m = upb_DefPool_FindMessageByName(self->symtab, name); + if (m == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + m = upb_DefPool_FindMessageByName(self->symtab, name); + } + if (m == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name); + } + + return PyUpb_Descriptor_Get(m); +} + +// Splits a dotted symbol like foo.bar.baz on the last dot. Returns the portion +// after the last dot (baz) and updates `*parent_size` to the length of the +// parent (foo.bar). Returns NULL if no dots were present. +static const char* PyUpb_DescriptorPool_SplitSymbolName(const char* sym, + size_t* parent_size) { + const char* last_dot = strrchr(sym, '.'); + if (!last_dot) return NULL; + *parent_size = last_dot - sym; + return last_dot + 1; +} + +/* + * PyUpb_DescriptorPool_FindFieldByName() + * + * Implements: + * DescriptorPool.FindFieldByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindFieldByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + size_t parent_size; + const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size); + const upb_FieldDef* f = NULL; + if (child) { + const upb_MessageDef* parent = + upb_DefPool_FindMessageByNameWithSize(self->symtab, name, parent_size); + if (parent == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + parent = upb_DefPool_FindMessageByNameWithSize(self->symtab, name, + parent_size); + } + if (parent) { + f = upb_MessageDef_FindFieldByName(parent, child); + } + } + + if (!f) { + return PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name); + } + + return PyUpb_FieldDescriptor_Get(f); +} + +/* + * PyUpb_DescriptorPool_FindEnumTypeByName() + * + * Implements: + * DescriptorPool.FindEnumTypeByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindEnumTypeByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_EnumDef* e = upb_DefPool_FindEnumByName(self->symtab, name); + if (e == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + e = upb_DefPool_FindEnumByName(self->symtab, name); + } + if (e == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name); + } + + return PyUpb_EnumDescriptor_Get(e); +} + +/* + * PyUpb_DescriptorPool_FindOneofByName() + * + * Implements: + * DescriptorPool.FindOneofByName(self, name) + */ +static PyObject* PyUpb_DescriptorPool_FindOneofByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + size_t parent_size; + const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size); + + if (child) { + const upb_MessageDef* parent = + upb_DefPool_FindMessageByNameWithSize(self->symtab, name, parent_size); + if (parent == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + parent = upb_DefPool_FindMessageByNameWithSize(self->symtab, name, + parent_size); + } + if (parent) { + const upb_OneofDef* o = upb_MessageDef_FindOneofByName(parent, child); + return PyUpb_OneofDescriptor_Get(o); + } + } + + return PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name); +} + +static PyObject* PyUpb_DescriptorPool_FindServiceByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_ServiceDef* s = upb_DefPool_FindServiceByName(self->symtab, name); + if (s == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + s = upb_DefPool_FindServiceByName(self->symtab, name); + } + if (s == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find service %.200s", name); + } + + return PyUpb_ServiceDescriptor_Get(s); +} + +static PyObject* PyUpb_DescriptorPool_FindMethodByName(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + size_t parent_size; + const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size); + + if (!child) goto err; + const upb_ServiceDef* parent = + upb_DefPool_FindServiceByNameWithSize(self->symtab, name, parent_size); + if (parent == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + parent = + upb_DefPool_FindServiceByNameWithSize(self->symtab, name, parent_size); + } + if (!parent) goto err; + const upb_MethodDef* m = upb_ServiceDef_FindMethodByName(parent, child); + if (!m) goto err; + return PyUpb_MethodDescriptor_Get(m); + +err: + return PyErr_Format(PyExc_KeyError, "Couldn't find method %.200s", name); +} + +static PyObject* PyUpb_DescriptorPool_FindFileContainingSymbol(PyObject* _self, + PyObject* arg) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + + const char* name = PyUpb_VerifyStrData(arg); + if (!name) return NULL; + + const upb_FileDef* f = + upb_DefPool_FindFileContainingSymbol(self->symtab, name); + if (f == NULL && self->db) { + if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL; + f = upb_DefPool_FindFileContainingSymbol(self->symtab, name); + } + if (f == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find symbol %.200s", name); + } + + return PyUpb_FileDescriptor_Get(f); +} + +static PyObject* PyUpb_DescriptorPool_FindExtensionByNumber(PyObject* _self, + PyObject* args) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + PyObject* message_descriptor; + int number; + if (!PyArg_ParseTuple(args, "Oi", &message_descriptor, &number)) { + return NULL; + } + + const upb_FieldDef* f = upb_DefPool_FindExtensionByNumber( + self->symtab, PyUpb_Descriptor_GetDef(message_descriptor), number); + if (f == NULL) { + return PyErr_Format(PyExc_KeyError, "Couldn't find Extension %d", number); + } + + return PyUpb_FieldDescriptor_Get(f); +} + +static PyObject* PyUpb_DescriptorPool_FindAllExtensions(PyObject* _self, + PyObject* msg_desc) { + PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self; + const upb_MessageDef* m = PyUpb_Descriptor_GetDef(msg_desc); + size_t n; + const upb_FieldDef** ext = upb_DefPool_GetAllExtensions(self->symtab, m, &n); + PyObject* ret = PyList_New(n); + if (!ret) goto done; + for (size_t i = 0; i < n; i++) { + PyObject* field = PyUpb_FieldDescriptor_Get(ext[i]); + if (!field) { + Py_DECREF(ret); + ret = NULL; + goto done; + } + PyList_SetItem(ret, i, field); + } +done: + free(ext); + return ret; +} + +static PyMethodDef PyUpb_DescriptorPool_Methods[] = { + {"Add", PyUpb_DescriptorPool_Add, METH_O, + "Adds the FileDescriptorProto and its types to this pool."}, + {"AddSerializedFile", PyUpb_DescriptorPool_AddSerializedFile, METH_O, + "Adds a serialized FileDescriptorProto to this pool."}, + {"FindFileByName", PyUpb_DescriptorPool_FindFileByName, METH_O, + "Searches for a file descriptor by its .proto name."}, + {"FindMessageTypeByName", PyUpb_DescriptorPool_FindMessageTypeByName, + METH_O, "Searches for a message descriptor by full name."}, + {"FindFieldByName", PyUpb_DescriptorPool_FindFieldByName, METH_O, + "Searches for a field descriptor by full name."}, + {"FindExtensionByName", PyUpb_DescriptorPool_FindExtensionByName, METH_O, + "Searches for extension descriptor by full name."}, + {"FindEnumTypeByName", PyUpb_DescriptorPool_FindEnumTypeByName, METH_O, + "Searches for enum type descriptor by full name."}, + {"FindOneofByName", PyUpb_DescriptorPool_FindOneofByName, METH_O, + "Searches for oneof descriptor by full name."}, + {"FindServiceByName", PyUpb_DescriptorPool_FindServiceByName, METH_O, + "Searches for service descriptor by full name."}, + {"FindMethodByName", PyUpb_DescriptorPool_FindMethodByName, METH_O, + "Searches for method descriptor by full name."}, + {"FindFileContainingSymbol", PyUpb_DescriptorPool_FindFileContainingSymbol, + METH_O, "Gets the FileDescriptor containing the specified symbol."}, + {"FindExtensionByNumber", PyUpb_DescriptorPool_FindExtensionByNumber, + METH_VARARGS, "Gets the extension descriptor for the given number."}, + {"FindAllExtensions", PyUpb_DescriptorPool_FindAllExtensions, METH_O, + "Gets all known extensions of the given message descriptor."}, + {NULL}}; + +static PyType_Slot PyUpb_DescriptorPool_Slots[] = { + {Py_tp_clear, PyUpb_DescriptorPool_Clear}, + {Py_tp_dealloc, PyUpb_DescriptorPool_Dealloc}, + {Py_tp_methods, PyUpb_DescriptorPool_Methods}, + {Py_tp_new, PyUpb_DescriptorPool_New}, + {Py_tp_traverse, PyUpb_DescriptorPool_Traverse}, + {0, NULL}}; + +static PyType_Spec PyUpb_DescriptorPool_Spec = { + PYUPB_MODULE_NAME ".DescriptorPool", + sizeof(PyUpb_DescriptorPool), + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + PyUpb_DescriptorPool_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +bool PyUpb_InitDescriptorPool(PyObject* m) { + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + PyTypeObject* descriptor_pool_type = + PyUpb_AddClass(m, &PyUpb_DescriptorPool_Spec); + + if (!descriptor_pool_type) return false; + + state->default_pool = PyUpb_DescriptorPool_DoCreateWithCache( + descriptor_pool_type, NULL, state->obj_cache); + return state->default_pool && + PyModule_AddObject(m, "default_pool", state->default_pool) == 0; +}
diff --git a/python/descriptor_pool.h b/python/descriptor_pool.h new file mode 100644 index 0000000..ae50ef0 --- /dev/null +++ b/python/descriptor_pool.h
@@ -0,0 +1,51 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_DESCRIPTOR_POOL_H__ +#define PYUPB_DESCRIPTOR_POOL_H__ + +#include <stdbool.h> + +#include "protobuf.h" + +// Returns a Python wrapper object for the given symtab. The symtab must have +// been created from a Python DescriptorPool originally. +PyObject* PyUpb_DescriptorPool_Get(const upb_DefPool* symtab); + +// Given a Python DescriptorPool, returns the underlying symtab. +upb_DefPool* PyUpb_DescriptorPool_GetSymtab(PyObject* pool); + +// Returns the default DescriptorPool (a global singleton). +PyObject* PyUpb_DescriptorPool_GetDefaultPool(void); + +// Module-level init. +bool PyUpb_InitDescriptorPool(PyObject* m); + +#endif // PYUPB_DESCRIPTOR_POOL_H__
diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel new file mode 100644 index 0000000..aa03fbe --- /dev/null +++ b/python/dist/BUILD.bazel
@@ -0,0 +1,453 @@ +# Copyright (c) 2009-2022, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@rules_python//python:packaging.bzl", "py_wheel") +load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") +load("//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION") +load("//bazel:py_proto_library.bzl", "py_proto_library") +load("@bazel_skylib//lib:selects.bzl", "selects") +load(":dist.bzl", "py_dist", "py_dist_module") + +licenses(["notice"]) + +py_dist_module( + name = "message_mod", + extension = "//python:_message_binary", + module_name = "google._upb._message", +) + +py_proto_library( + name = "well_known_proto_py_pb2", + deps = [ + "//:any_proto", + "//:api_proto", + "//:descriptor_proto", + "//:duration_proto", + "//:empty_proto", + "//:field_mask_proto", + "//:source_context_proto", + "//:struct_proto", + "//:timestamp_proto", + "//:type_proto", + "//:wrappers_proto", + ], +) + +py_proto_library( + name = "plugin_py_pb2", + deps = ["//:compiler_plugin_proto"], +) + +config_setting( + name = "linux_aarch64_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "linux-aarch_64"}, +) + +config_setting( + name = "linux_aarch64_local", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +config_setting( + name = "linux_x86_64_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "linux-x86_64"}, +) + +config_setting( + name = "linux_x86_64_local", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +config_setting( + name = "osx_x86_64_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "osx-x86_64"}, +) + +config_setting( + name = "osx_x86_64_local", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +selects.config_setting_group( + name = "osx_x86_64", + match_any = [ + ":osx_x86_64_release", + ":osx_x86_64_local", + ], +) + +config_setting( + name = "osx_aarch64_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "osx-aarch_64"}, +) + +config_setting( + name = "osx_aarch64_local", + constraint_values = [ + "@platforms//os:osx", + "@platforms//cpu:aarch64", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +selects.config_setting_group( + name = "osx_aarch64", + match_any = [ + ":osx_aarch64_release", + ":osx_aarch64_local", + ], +) + +config_setting( + name = "osx_universal2", + values = {"cpu": "osx-universal2"}, +) + +config_setting( + name = "windows_x86_32_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "win32"}, +) + +config_setting( + name = "windows_x86_32_local", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_32", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +selects.config_setting_group( + name = "windows_x86_32", + match_any = [ + ":windows_x86_32_release", + ":windows_x86_32_local", + ], +) + +config_setting( + name = "windows_x86_64_release", + flag_values = { + "//toolchain:release": "True", + }, + values = {"cpu": "win64"}, +) + +config_setting( + name = "windows_x86_64_local", + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + flag_values = { + "//toolchain:release": "False", + }, +) + +selects.config_setting_group( + name = "windows_x86_64", + match_any = [ + ":windows_x86_64_release", + ":windows_x86_64_local", + ], +) + +pkg_files( + name = "generated_wkt", + srcs = [ + ":well_known_proto_py_pb2", + "//upb:descriptor_upb_minitable_proto", + "//upb:descriptor_upb_proto", + "//upb:descriptor_upb_proto_reflection", + ], + prefix = "google/protobuf", +) + +pkg_files( + name = "generated_wkt_compiler", + srcs = [ + ":plugin_py_pb2", + ], + prefix = "google/protobuf/compiler", +) + +pkg_files( + name = "utf8_range_source_files", + srcs = ["@utf8_range//:utf8_range_srcs"], + prefix = "utf8_range", +) + +pkg_files( + name = "dist_source_files", + srcs = [ + "MANIFEST.in", + "setup.py", + ], +) + +# Passing filegroups to pkg_tar directly results in incorrect +# `protobuf/external/upb/` directory structure when built from the protobuf +# repo. This can be removed once repositories are merged. +pkg_files( + name = "filegroup_source_files", + srcs = [ + "//:LICENSE", + "//python:message_srcs", + "//upb:source_files", + "//upb/base:source_files", + "//upb/collections:source_files", + "//upb/hash:source_files", + "//upb/lex:source_files", + "//upb/mem:source_files", + "//upb/message:source_files", + "//upb/mini_descriptor:source_files", + "//upb/mini_table:source_files", + "//upb/port:source_files", + "//upb/text:source_files", + "//upb/util:source_files", + "//upb/wire:source_files", + ], + strip_prefix = strip_prefix.from_root(""), +) + +# NOTE: This package currently only works for macos and ubuntu, MSVC users +# should use a binary wheel. +pkg_tar( + name = "source_tarball", + srcs = [ + ":dist_source_files", + ":filegroup_source_files", + ":generated_wkt", + ":generated_wkt_compiler", + ":utf8_range_source_files", + "//python:python_source_files", + ], + extension = "tar.gz", + package_dir = "protobuf", + package_file_name = "protobuf.tar.gz", + strip_prefix = ".", + target_compatible_with = select({ + "@system_python//:none": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) + +genrule( + name = "source_wheel", + srcs = [":source_tarball"], + outs = ["protobuf-%s.tar.gz" % PROTOBUF_PYTHON_VERSION], + cmd = """ + set -eux + tar -xzvf $(location :source_tarball) + cd protobuf/ + python3 setup.py sdist + cd .. + mv protobuf/dist/*.tar.gz $@ + """, + target_compatible_with = select({ + "@system_python//:none": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +) + +py_wheel( + name = "binary_wheel", + abi = select({ + "//python:full_api_3.7": "cp37m", + "//python:full_api_3.8": "cp38", + "//python:full_api_3.9": "cp39", + "//conditions:default": "abi3", + }), + author = "protobuf@googlegroups.com", + author_email = "protobuf@googlegroups.com", + classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], + distribution = "protobuf", + extra_distinfo_files = { + "//:LICENSE": "LICENSE", + }, + homepage = "https://developers.google.com/protocol-buffers/", + license = "3-Clause BSD License", + platform = select({ + ":linux_x86_64_local": "linux_x86_64", + ":linux_x86_64_release": "manylinux2014_x86_64", + ":linux_aarch64_local": "linux_aarch64", + ":linux_aarch64_release": "manylinux2014_aarch64", + ":osx_universal2": "macosx_10_9_universal2", + ":osx_aarch64": "macosx_11_0_arm64", + ":windows_x86_32": "win32", + ":windows_x86_64": "win_amd64", + "//conditions:default": "any", + }), + python_requires = ">=3.7", + python_tag = selects.with_or({ + ("//python:limited_api_3.7", "//python:full_api_3.7"): "cp37", + "//python:full_api_3.8": "cp38", + "//python:full_api_3.9": "cp39", + "//python:limited_api_3.10": "cp310", + "//conditions:default": "cp" + SYSTEM_PYTHON_VERSION, + }), + strip_path_prefixes = [ + "python/dist/", + "python/", + "src/", + ], + target_compatible_with = select({ + "@system_python//:none": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + version = PROTOBUF_PYTHON_VERSION, + deps = [ + ":message_mod", + ":plugin_py_pb2", + ":well_known_proto_py_pb2", + "//:python_srcs", + ], +) + +py_wheel( + name = "pure_python_wheel", + abi = "none", + author = "protobuf@googlegroups.com", + author_email = "protobuf@googlegroups.com", + classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], + distribution = "protobuf", + extra_distinfo_files = { + "//:LICENSE": "LICENSE", + }, + homepage = "https://developers.google.com/protocol-buffers/", + license = "3-Clause BSD License", + platform = "any", + python_requires = ">=3.7", + python_tag = "py3", + strip_path_prefixes = [ + "python/", + "src/", + ], + target_compatible_with = select({ + "@system_python//:none": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + version = PROTOBUF_PYTHON_VERSION, + deps = [ + ":plugin_py_pb2", + ":well_known_proto_py_pb2", + "//:python_srcs", + ], +) + +py_wheel( + name = "test_wheel", + testonly = True, + abi = "none", + distribution = "protobuftests", + extra_distinfo_files = { + "//:LICENSE": "LICENSE", + }, + platform = "any", + python_tag = "py3", + strip_path_prefixes = [ + "python/", + "src/", + ], + target_compatible_with = select({ + "@system_python//:none": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + version = PROTOBUF_PYTHON_VERSION, + deps = [ + "//:python_common_test_protos", + "//:python_specific_test_protos", + "//:python_test_srcs", + "//python/pb_unit_tests:test_files", + "//src/google/protobuf:testdata", + ], +) + +py_dist( + name = "dist", + binary_wheel = ":binary_wheel", + full_api_cpus = [ + # TODO: fix win32 build + "win32", + "win64", + ], + # Windows needs version-specific wheels until 3.10. + full_api_versions = [ + "37", + "38", + "39", + ], + # Limited API: these wheels will satisfy any Python version >= the + # given version. + # + # Technically the limited API doesn't have the functions we need until + # 3.10, but on Linux we can get away with using 3.7 (see ../python_api.h for + # details). + limited_api_wheels = { + # TODO: fix win32 build + "win32": "310", + "win64": "310", + "linux-x86_64": "37", + "linux-aarch_64": "37", + "osx-universal2": "37", + }, + pure_python_wheel = ":pure_python_wheel", + tags = ["manual"], +)
diff --git a/python/dist/MANIFEST.in b/python/dist/MANIFEST.in new file mode 100644 index 0000000..1b61936 --- /dev/null +++ b/python/dist/MANIFEST.in
@@ -0,0 +1,2 @@ +global-include *.h +global-include *.inc \ No newline at end of file
diff --git a/python/dist/dist.bzl b/python/dist/dist.bzl new file mode 100644 index 0000000..75c21c3 --- /dev/null +++ b/python/dist/dist.bzl
@@ -0,0 +1,193 @@ +"""Rules to create python distribution files and properly name them""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") + +def _get_suffix(limited_api, python_version, cpu): + """Computes an ABI version tag for an extension module per PEP 3149.""" + if "win32" in cpu or "win64" in cpu: + if limited_api: + return ".pyd" + if "win32" in cpu: + abi = "win32" + elif "win64" in cpu: + abi = "win_amd64" + else: + fail("Unsupported CPU: " + cpu) + return ".cp{}-{}.{}".format(python_version, abi, "pyd") + + if python_version == "system": + python_version = SYSTEM_PYTHON_VERSION + if int(python_version) < 38: + python_version += "m" + abis = { + "darwin_arm64": "darwin", + "darwin_x86_64": "darwin", + "darwin": "darwin", + "osx-x86_64": "darwin", + "osx-aarch_64": "darwin", + "linux-aarch_64": "aarch64-linux-gnu", + "linux-x86_64": "x86_64-linux-gnu", + "k8": "x86_64-linux-gnu", + } + + return ".cpython-{}-{}.{}".format( + python_version, + abis[cpu], + "so" if limited_api else "abi3.so", + ) + elif limited_api: + return ".abi3.so" + + fail("Unsupported combination of flags") + +def _declare_module_file(ctx, module_name, python_version, limited_api): + """Declares an output file for a Python module with this name, version, and limited api.""" + base_filename = module_name.replace(".", "/") + suffix = _get_suffix( + python_version = python_version, + limited_api = limited_api, + cpu = ctx.var["TARGET_CPU"], + ) + filename = base_filename + suffix + return ctx.actions.declare_file(filename) + +# -------------------------------------------------------------------------------------------------- +# py_dist_module() +# +# Creates a Python binary extension module that is ready for distribution. +# +# py_dist_module( +# name = "message_mod", +# extension = "//python:_message_binary", +# module_name = "google._upb._message", +# ) +# +# In the simple case, this simply involves copying the input file to the proper filename for +# our current configuration (module_name, cpu, python_version, limited_abi). +# +# For multiarch platforms (osx-universal2), we must combine binaries for multiple architectures +# into a single output binary using the "llvm-lipo" tool. A config transition depends on multiple +# architectures to get us the input files we need. + +def _py_multiarch_transition_impl(settings, attr): + if settings["//command_line_option:cpu"] == "osx-universal2": + return [{"//command_line_option:cpu": cpu} for cpu in ["osx-aarch_64", "osx-x86_64"]] + else: + return settings + +_py_multiarch_transition = transition( + implementation = _py_multiarch_transition_impl, + inputs = ["//command_line_option:cpu"], + outputs = ["//command_line_option:cpu"], +) + +def _py_dist_module_impl(ctx): + output_file = _declare_module_file( + ctx = ctx, + module_name = ctx.attr.module_name, + python_version = ctx.attr._python_version[BuildSettingInfo].value, + limited_api = ctx.attr._limited_api[BuildSettingInfo].value, + ) + if len(ctx.attr.extension) == 1: + src = ctx.attr.extension[0][DefaultInfo].files.to_list()[0] + ctx.actions.run( + executable = "cp", + arguments = [src.path, output_file.path], + inputs = [src], + outputs = [output_file], + ) + return [ + DefaultInfo(files = depset([output_file])), + ] + else: + srcs = [mod[DefaultInfo].files.to_list()[0] for mod in ctx.attr.extension] + ctx.actions.run( + executable = "/usr/local/bin/llvm-lipo", + arguments = ["-create", "-output", output_file.path] + [src.path for src in srcs], + inputs = srcs, + outputs = [output_file], + ) + return [ + DefaultInfo(files = depset([output_file])), + ] + +py_dist_module = rule( + output_to_genfiles = True, + implementation = _py_dist_module_impl, + attrs = { + "module_name": attr.string(mandatory = True), + "extension": attr.label( + mandatory = True, + cfg = _py_multiarch_transition, + ), + "_limited_api": attr.label(default = "//python:limited_api"), + "_python_version": attr.label(default = "//python:python_version"), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +# -------------------------------------------------------------------------------------------------- +# py_dist() +# +# A rule that builds a collection of binary wheels, using transitions to depend on many different +# python versions and cpus. + +def _py_dist_transition_impl(settings, attr): + _ignore = (settings) # @unused + transitions = [] + + for cpu, version in attr.limited_api_wheels.items(): + transitions.append({ + "//command_line_option:cpu": cpu, + "//python:python_version": version, + "//python:limited_api": True, + }) + + for version in attr.full_api_versions: + for cpu in attr.full_api_cpus: + transitions.append({ + "//command_line_option:cpu": cpu, + "//python:python_version": version, + "//python:limited_api": False, + }) + + return transitions + +_py_dist_transition = transition( + implementation = _py_dist_transition_impl, + inputs = [], + outputs = [ + "//command_line_option:cpu", + "//python:python_version", + "//python:limited_api", + ], +) + +def _py_dist_impl(ctx): + binary_files = [dep[DefaultInfo].files for dep in ctx.attr.binary_wheel] + pure_python_files = [ctx.attr.pure_python_wheel[DefaultInfo].files] + return [ + DefaultInfo(files = depset( + transitive = binary_files + pure_python_files, + )), + ] + +py_dist = rule( + implementation = _py_dist_impl, + attrs = { + "binary_wheel": attr.label( + mandatory = True, + cfg = _py_dist_transition, + ), + "pure_python_wheel": attr.label(mandatory = True), + "limited_api_wheels": attr.string_dict(), + "full_api_versions": attr.string_list(), + "full_api_cpus": attr.string_list(), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +)
diff --git a/python/dist/setup.py b/python/dist/setup.py new file mode 100755 index 0000000..b4fc21a --- /dev/null +++ b/python/dist/setup.py
@@ -0,0 +1,80 @@ +#! /usr/bin/env python +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd +# +# See README for usage instructions. + +import glob +import os +import sys +import sysconfig + +# We must use setuptools, not distutils, because we need to use the +# namespace_packages option for the "google" package. +from setuptools import setup, Extension, find_packages + + +def GetVersion(): + """Reads and returns the version from google/protobuf/__init__.py. + + Do not import google.protobuf.__init__ directly, because an installed + protobuf library may be loaded instead. + + Returns: + The version. + """ + + with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: + file_globals = {} + exec(version_file.read(), file_globals) # pylint:disable=exec-used + return file_globals["__version__"] + + +current_dir = os.path.dirname(os.path.abspath(__file__)) +extra_link_args = [] + +if sys.platform.startswith('win'): + extra_link_args = ['-static'] + +setup( + name='protobuf', + version=GetVersion(), + description='Protocol Buffers', + download_url='https://github.com/protocolbuffers/protobuf/releases', + long_description="Protocol Buffers are Google's data interchange format", + url='https://developers.google.com/protocol-buffers/', + project_urls={ + 'Source': 'https://github.com/protocolbuffers/protobuf', + }, + maintainer='protobuf@googlegroups.com', + maintainer_email='protobuf@googlegroups.com', + license='BSD-3-Clause', + classifiers=[ + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + ], + namespace_packages=['google'], + packages=find_packages(), + install_requires=[], + ext_modules=[ + Extension( + 'google._upb._message', + glob.glob('google/protobuf/*.c') + + glob.glob('python/*.c') + + glob.glob('upb/**/*.c', recursive=True) + + glob.glob('utf8_range/*.c'), + include_dirs=[current_dir, os.path.join(current_dir, 'utf8_range')], + language='c', + extra_link_args=extra_link_args, + ) + ], + python_requires='>=3.7', +)
diff --git a/python/extension_dict.c b/python/extension_dict.c new file mode 100644 index 0000000..d4b4dda --- /dev/null +++ b/python/extension_dict.c
@@ -0,0 +1,256 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/extension_dict.h" + +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/reflection/def.h" + +// ----------------------------------------------------------------------------- +// ExtensionDict +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + PyObject* msg; // Owning ref to our parent pessage. +} PyUpb_ExtensionDict; + +PyObject* PyUpb_ExtensionDict_New(PyObject* msg) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyUpb_ExtensionDict* ext_dict = + (void*)PyType_GenericAlloc(state->extension_dict_type, 0); + ext_dict->msg = msg; + Py_INCREF(ext_dict->msg); + return &ext_dict->ob_base; +} + +static PyObject* PyUpb_ExtensionDict_FindExtensionByName(PyObject* _self, + PyObject* key) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + const char* name = PyUpb_GetStrData(key); + if (!name) { + PyErr_Format(PyExc_TypeError, "_FindExtensionByName expect a str"); + return NULL; + } + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(self->msg); + const upb_FileDef* file = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(file); + const upb_FieldDef* ext = upb_DefPool_FindExtensionByName(symtab, name); + if (ext) { + return PyUpb_FieldDescriptor_Get(ext); + } else { + Py_RETURN_NONE; + } +} + +static PyObject* PyUpb_ExtensionDict_FindExtensionByNumber(PyObject* _self, + PyObject* arg) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(self->msg); + const upb_MiniTable* l = upb_MessageDef_MiniTable(m); + const upb_FileDef* file = upb_MessageDef_File(m); + const upb_DefPool* symtab = upb_FileDef_Pool(file); + const upb_ExtensionRegistry* reg = upb_DefPool_ExtensionRegistry(symtab); + int64_t number = PyLong_AsLong(arg); + if (number == -1 && PyErr_Occurred()) return NULL; + const upb_MiniTableExtension* ext = + (upb_MiniTableExtension*)upb_ExtensionRegistry_Lookup(reg, l, number); + if (ext) { + const upb_FieldDef* f = upb_DefPool_FindExtensionByMiniTable(symtab, ext); + return PyUpb_FieldDescriptor_Get(f); + } else { + Py_RETURN_NONE; + } +} + +static void PyUpb_ExtensionDict_Dealloc(PyUpb_ExtensionDict* self) { + PyUpb_Message_ClearExtensionDict(self->msg); + Py_DECREF(self->msg); + PyUpb_Dealloc(self); +} + +static PyObject* PyUpb_ExtensionDict_RichCompare(PyObject* _self, + PyObject* _other, int opid) { + // Only equality comparisons are implemented. + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + bool equals = false; + if (PyObject_TypeCheck(_other, Py_TYPE(_self))) { + PyUpb_ExtensionDict* other = (PyUpb_ExtensionDict*)_other; + equals = self->msg == other->msg; + } + bool ret = opid == Py_EQ ? equals : !equals; + return PyBool_FromLong(ret); +} + +static int PyUpb_ExtensionDict_Contains(PyObject* _self, PyObject* key) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + const upb_FieldDef* f = PyUpb_Message_GetExtensionDef(self->msg, key); + if (!f) return -1; + upb_Message* msg = PyUpb_Message_GetIfReified(self->msg); + if (!msg) return 0; + if (upb_FieldDef_IsRepeated(f)) { + upb_MessageValue val = upb_Message_GetFieldByDef(msg, f); + return upb_Array_Size(val.array_val) > 0; + } else { + return upb_Message_HasFieldByDef(msg, f); + } +} + +static Py_ssize_t PyUpb_ExtensionDict_Length(PyObject* _self) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + upb_Message* msg = PyUpb_Message_GetIfReified(self->msg); + return msg ? upb_Message_ExtensionCount(msg) : 0; +} + +static PyObject* PyUpb_ExtensionDict_Subscript(PyObject* _self, PyObject* key) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + const upb_FieldDef* f = PyUpb_Message_GetExtensionDef(self->msg, key); + if (!f) return NULL; + return PyUpb_Message_GetFieldValue(self->msg, f); +} + +static int PyUpb_ExtensionDict_AssignSubscript(PyObject* _self, PyObject* key, + PyObject* val) { + PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self; + const upb_FieldDef* f = PyUpb_Message_GetExtensionDef(self->msg, key); + if (!f) return -1; + if (val) { + return PyUpb_Message_SetFieldValue(self->msg, f, val, PyExc_TypeError); + } else { + PyUpb_Message_DoClearField(self->msg, f); + return 0; + } +} + +static PyObject* PyUpb_ExtensionIterator_New(PyObject* _ext_dict); + +static PyMethodDef PyUpb_ExtensionDict_Methods[] = { + {"_FindExtensionByName", PyUpb_ExtensionDict_FindExtensionByName, METH_O, + "Finds an extension by name."}, + {"_FindExtensionByNumber", PyUpb_ExtensionDict_FindExtensionByNumber, + METH_O, "Finds an extension by number."}, + {NULL, NULL}, +}; + +static PyType_Slot PyUpb_ExtensionDict_Slots[] = { + {Py_tp_dealloc, PyUpb_ExtensionDict_Dealloc}, + {Py_tp_methods, PyUpb_ExtensionDict_Methods}, + //{Py_tp_getset, PyUpb_ExtensionDict_Getters}, + //{Py_tp_hash, PyObject_HashNotImplemented}, + {Py_tp_richcompare, PyUpb_ExtensionDict_RichCompare}, + {Py_tp_iter, PyUpb_ExtensionIterator_New}, + {Py_sq_contains, PyUpb_ExtensionDict_Contains}, + {Py_sq_length, PyUpb_ExtensionDict_Length}, + {Py_mp_length, PyUpb_ExtensionDict_Length}, + {Py_mp_subscript, PyUpb_ExtensionDict_Subscript}, + {Py_mp_ass_subscript, PyUpb_ExtensionDict_AssignSubscript}, + {0, NULL}}; + +static PyType_Spec PyUpb_ExtensionDict_Spec = { + PYUPB_MODULE_NAME ".ExtensionDict", // tp_name + sizeof(PyUpb_ExtensionDict), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ExtensionDict_Slots, +}; + +// ----------------------------------------------------------------------------- +// ExtensionIterator +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + PyObject* msg; + size_t iter; +} PyUpb_ExtensionIterator; + +static PyObject* PyUpb_ExtensionIterator_New(PyObject* _ext_dict) { + PyUpb_ExtensionDict* ext_dict = (PyUpb_ExtensionDict*)_ext_dict; + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyUpb_ExtensionIterator* iter = + (void*)PyType_GenericAlloc(state->extension_iterator_type, 0); + if (!iter) return NULL; + iter->msg = ext_dict->msg; + iter->iter = kUpb_Message_Begin; + Py_INCREF(iter->msg); + return &iter->ob_base; +} + +static void PyUpb_ExtensionIterator_Dealloc(void* _self) { + PyUpb_ExtensionIterator* self = (PyUpb_ExtensionIterator*)_self; + Py_DECREF(self->msg); + PyUpb_Dealloc(_self); +} + +PyObject* PyUpb_ExtensionIterator_IterNext(PyObject* _self) { + PyUpb_ExtensionIterator* self = (PyUpb_ExtensionIterator*)_self; + upb_Message* msg = PyUpb_Message_GetIfReified(self->msg); + if (!msg) return NULL; + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(self->msg); + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m)); + while (true) { + const upb_FieldDef* f; + upb_MessageValue val; + if (!upb_Message_Next(msg, m, symtab, &f, &val, &self->iter)) return NULL; + if (upb_FieldDef_IsExtension(f)) return PyUpb_FieldDescriptor_Get(f); + } +} + +static PyType_Slot PyUpb_ExtensionIterator_Slots[] = { + {Py_tp_dealloc, PyUpb_ExtensionIterator_Dealloc}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, PyUpb_ExtensionIterator_IterNext}, + {0, NULL}}; + +static PyType_Spec PyUpb_ExtensionIterator_Spec = { + PYUPB_MODULE_NAME ".ExtensionIterator", // tp_name + sizeof(PyUpb_ExtensionIterator), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_ExtensionIterator_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +bool PyUpb_InitExtensionDict(PyObject* m) { + PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m); + + s->extension_dict_type = PyUpb_AddClass(m, &PyUpb_ExtensionDict_Spec); + s->extension_iterator_type = PyUpb_AddClass(m, &PyUpb_ExtensionIterator_Spec); + + return s->extension_dict_type && s->extension_iterator_type; +}
diff --git a/python/extension_dict.h b/python/extension_dict.h new file mode 100644 index 0000000..99d2add --- /dev/null +++ b/python/extension_dict.h
@@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_EXTENSION_DICT_H__ +#define PYUPB_EXTENSION_DICT_H__ + +#include <stdbool.h> + +#include "python/python_api.h" + +PyObject* PyUpb_ExtensionDict_New(PyObject* msg); + +bool PyUpb_InitExtensionDict(PyObject* m); + +#endif // PYUPB_EXTENSION_DICT_H__
diff --git a/python/google/protobuf/internal/numpy/BUILD.bazel b/python/google/protobuf/internal/numpy/BUILD.bazel index 5c1c166..ffdc4e5 100644 --- a/python/google/protobuf/internal/numpy/BUILD.bazel +++ b/python/google/protobuf/internal/numpy/BUILD.bazel
@@ -6,18 +6,18 @@ # TODO: b/278896688 - Remove this target and replace with py_library exports_files([ - "__init__.py", - "numpy_test.py", + "__init__.py", + "numpy_test.py", ]) internal_py_test( name = "numpy_test", srcs = ["numpy_test.py"], + visibility = [ + "//python:__pkg__", + "//python/pb_unit_tests:__pkg__", + ], deps = [ requirement("numpy"), ], - visibility = [ - "//python:__pkg__", - "//upb/python/pb_unit_tests:__pkg__", - ] )
diff --git a/python/map.c b/python/map.c new file mode 100644 index 0000000..bd9022d --- /dev/null +++ b/python/map.c
@@ -0,0 +1,529 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/map.h" + +#include "python/convert.h" +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/collections/map.h" +#include "upb/reflection/def.h" + +// ----------------------------------------------------------------------------- +// MapContainer +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + PyObject* arena; + // The field descriptor (upb_FieldDef*). + // The low bit indicates whether the container is reified (see ptr below). + // - low bit set: repeated field is a stub (empty map, no underlying data). + // - low bit clear: repeated field is reified (points to upb_Array). + uintptr_t field; + union { + PyObject* parent; // stub: owning pointer to parent message. + upb_Map* map; // reified: the data for this array. + } ptr; + int version; +} PyUpb_MapContainer; + +static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map); + +static bool PyUpb_MapContainer_IsStub(PyUpb_MapContainer* self) { + return self->field & 1; +} + +// If the map is reified, returns it. Otherwise, returns NULL. +// If NULL is returned, the object is empty and has no underlying data. +static upb_Map* PyUpb_MapContainer_GetIfReified(PyUpb_MapContainer* self) { + return PyUpb_MapContainer_IsStub(self) ? NULL : self->ptr.map; +} + +static const upb_FieldDef* PyUpb_MapContainer_GetField( + PyUpb_MapContainer* self) { + return (const upb_FieldDef*)(self->field & ~(uintptr_t)1); +} + +static void PyUpb_MapContainer_Dealloc(void* _self) { + PyUpb_MapContainer* self = _self; + Py_DECREF(self->arena); + if (PyUpb_MapContainer_IsStub(self)) { + PyUpb_Message_CacheDelete(self->ptr.parent, + PyUpb_MapContainer_GetField(self)); + Py_DECREF(self->ptr.parent); + } else { + PyUpb_ObjCache_Delete(self->ptr.map); + } + PyUpb_Dealloc(_self); +} + +PyTypeObject* PyUpb_MapContainer_GetClass(const upb_FieldDef* f) { + assert(upb_FieldDef_IsMap(f)); + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + const upb_FieldDef* val = + upb_MessageDef_Field(upb_FieldDef_MessageSubDef(f), 1); + assert(upb_FieldDef_Number(val) == 2); + return upb_FieldDef_IsSubMessage(val) ? state->message_map_container_type + : state->scalar_map_container_type; +} + +PyObject* PyUpb_MapContainer_NewStub(PyObject* parent, const upb_FieldDef* f, + PyObject* arena) { + // We only create stubs when the parent is reified, by convention. However + // this is not an invariant: the parent could become reified at any time. + assert(PyUpb_Message_GetIfReified(parent) == NULL); + PyTypeObject* cls = PyUpb_MapContainer_GetClass(f); + PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0); + map->arena = arena; + map->field = (uintptr_t)f | 1; + map->ptr.parent = parent; + map->version = 0; + Py_INCREF(arena); + Py_INCREF(parent); + return &map->ob_base; +} + +void PyUpb_MapContainer_Reify(PyObject* _self, upb_Map* map) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + if (!map) { + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + map = upb_Map_New(arena, upb_FieldDef_CType(key_f), + upb_FieldDef_CType(val_f)); + } + PyUpb_ObjCache_Add(map, &self->ob_base); + Py_DECREF(self->ptr.parent); + self->ptr.map = map; // Overwrites self->ptr.parent. + self->field &= ~(uintptr_t)1; + assert(!PyUpb_MapContainer_IsStub(self)); +} + +void PyUpb_MapContainer_Invalidate(PyObject* obj) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)obj; + self->version++; +} + +upb_Map* PyUpb_MapContainer_EnsureReified(PyObject* _self) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + self->version++; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + if (map) return map; // Already writable. + + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + map = + upb_Map_New(arena, upb_FieldDef_CType(key_f), upb_FieldDef_CType(val_f)); + upb_MessageValue msgval = {.map_val = map}; + PyUpb_Message_SetConcreteSubobj(self->ptr.parent, f, msgval); + PyUpb_MapContainer_Reify((PyObject*)self, map); + return map; +} + +bool PyUpb_MapContainer_Set(PyUpb_MapContainer* self, upb_Map* map, + upb_MessageValue key, upb_MessageValue val, + upb_Arena* arena) { + switch (upb_Map_Insert(map, key, val, arena)) { + case kUpb_MapInsertStatus_Inserted: + return true; + case kUpb_MapInsertStatus_Replaced: + // We did not insert a new key, undo the previous invalidate. + self->version--; + return true; + case kUpb_MapInsertStatus_OutOfMemory: + return false; + } + return false; // Unreachable, silence compiler warning. +} + +int PyUpb_MapContainer_AssignSubscript(PyObject* _self, PyObject* key, + PyObject* val) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + upb_Map* map = PyUpb_MapContainer_EnsureReified(_self); + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + upb_MessageValue u_key, u_val; + if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return -1; + + if (val) { + if (!PyUpb_PyToUpb(val, val_f, &u_val, arena)) return -1; + if (!PyUpb_MapContainer_Set(self, map, u_key, u_val, arena)) return -1; + } else { + if (!upb_Map_Delete(map, u_key, NULL)) { + PyErr_Format(PyExc_KeyError, "Key not present in map"); + return -1; + } + } + return 0; +} + +PyObject* PyUpb_MapContainer_Subscript(PyObject* _self, PyObject* key) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + upb_MessageValue u_key, u_val; + if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL; + if (!map || !upb_Map_Get(map, u_key, &u_val)) { + map = PyUpb_MapContainer_EnsureReified(_self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + if (upb_FieldDef_IsSubMessage(val_f)) { + const upb_Message* m = upb_FieldDef_MessageSubDef(val_f); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + u_val.msg_val = upb_Message_New(layout, arena); + } else { + memset(&u_val, 0, sizeof(u_val)); + } + if (!PyUpb_MapContainer_Set(self, map, u_key, u_val, arena)) return false; + } + return PyUpb_UpbToPy(u_val, val_f, self->arena); +} + +PyObject* PyUpb_MapContainer_Contains(PyObject* _self, PyObject* key) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + if (!map) Py_RETURN_FALSE; + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + upb_MessageValue u_key; + if (!PyUpb_PyToUpb(key, key_f, &u_key, NULL)) return NULL; + if (upb_Map_Get(map, u_key, NULL)) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} + +PyObject* PyUpb_MapContainer_Clear(PyObject* _self, PyObject* key) { + upb_Map* map = PyUpb_MapContainer_EnsureReified(_self); + upb_Map_Clear(map); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_MapContainer_Get(PyObject* _self, PyObject* args, + PyObject* kwargs) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + static const char* kwlist[] = {"key", "default", NULL}; + PyObject* key; + PyObject* default_value = NULL; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", (char**)kwlist, &key, + &default_value)) { + return NULL; + } + + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + upb_MessageValue u_key, u_val; + if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL; + if (map && upb_Map_Get(map, u_key, &u_val)) { + return PyUpb_UpbToPy(u_val, val_f, self->arena); + } + if (default_value) { + Py_INCREF(default_value); + return default_value; + } + Py_RETURN_NONE; +} + +static PyObject* PyUpb_MapContainer_GetEntryClass(PyObject* _self, + PyObject* arg) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + return PyUpb_Descriptor_GetClass(entry_m); +} + +Py_ssize_t PyUpb_MapContainer_Length(PyObject* _self) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + return map ? upb_Map_Size(map) : 0; +} + +PyUpb_MapContainer* PyUpb_MapContainer_Check(PyObject* _self) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + if (!PyObject_TypeCheck(_self, state->message_map_container_type) && + !PyObject_TypeCheck(_self, state->scalar_map_container_type)) { + PyErr_Format(PyExc_TypeError, "Expected protobuf map, but got %R", _self); + return NULL; + } + return (PyUpb_MapContainer*)_self; +} + +int PyUpb_Message_InitMapAttributes(PyObject* map, PyObject* value, + const upb_FieldDef* f); + +static PyObject* PyUpb_MapContainer_MergeFrom(PyObject* _self, PyObject* _arg) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + + if (PyDict_Check(_arg)) { + return PyErr_Format(PyExc_AttributeError, "Merging of dict is not allowed"); + } + + if (PyUpb_Message_InitMapAttributes(_self, _arg, f) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* PyUpb_MapContainer_Repr(PyObject* _self) { + PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self; + upb_Map* map = PyUpb_MapContainer_GetIfReified(self); + PyObject* dict = PyDict_New(); + if (map) { + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + size_t iter = kUpb_Map_Begin; + upb_MessageValue map_key, map_val; + while (upb_Map_Next(map, &map_key, &map_val, &iter)) { + PyObject* key = PyUpb_UpbToPy(map_key, key_f, self->arena); + PyObject* val = PyUpb_UpbToPy(map_val, val_f, self->arena); + if (!key || !val) { + Py_XDECREF(key); + Py_XDECREF(val); + Py_DECREF(dict); + return NULL; + } + PyDict_SetItem(dict, key, val); + Py_DECREF(key); + Py_DECREF(val); + } + } + PyObject* repr = PyObject_Repr(dict); + Py_DECREF(dict); + return repr; +} + +PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_Map* map, + const upb_FieldDef* f, + PyObject* arena) { + PyUpb_MapContainer* ret = (void*)PyUpb_ObjCache_Get(map); + if (ret) return &ret->ob_base; + + PyTypeObject* cls = PyUpb_MapContainer_GetClass(f); + ret = (void*)PyType_GenericAlloc(cls, 0); + ret->arena = arena; + ret->field = (uintptr_t)f; + ret->ptr.map = map; + ret->version = 0; + Py_INCREF(arena); + PyUpb_ObjCache_Add(map, &ret->ob_base); + return &ret->ob_base; +} + +// ----------------------------------------------------------------------------- +// ScalarMapContainer +// ----------------------------------------------------------------------------- + +static PyMethodDef PyUpb_ScalarMapContainer_Methods[] = { + {"__contains__", PyUpb_MapContainer_Contains, METH_O, + "Tests whether a key is a member of the map."}, + {"clear", PyUpb_MapContainer_Clear, METH_NOARGS, + "Removes all elements from the map."}, + {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS, + "Gets the value for the given key if present, or otherwise a default"}, + {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS, + "Return the class used to build Entries of (key, value) pairs."}, + {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O, + "Merges a map into the current map."}, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +static PyType_Slot PyUpb_ScalarMapContainer_Slots[] = { + {Py_tp_dealloc, PyUpb_MapContainer_Dealloc}, + {Py_mp_length, PyUpb_MapContainer_Length}, + {Py_mp_subscript, PyUpb_MapContainer_Subscript}, + {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript}, + {Py_tp_methods, PyUpb_ScalarMapContainer_Methods}, + {Py_tp_iter, PyUpb_MapIterator_New}, + {Py_tp_repr, PyUpb_MapContainer_Repr}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_ScalarMapContainer_Spec = { + PYUPB_MODULE_NAME ".ScalarMapContainer", + sizeof(PyUpb_MapContainer), + 0, + Py_TPFLAGS_DEFAULT, + PyUpb_ScalarMapContainer_Slots, +}; + +// ----------------------------------------------------------------------------- +// MessageMapContainer +// ----------------------------------------------------------------------------- + +static PyMethodDef PyUpb_MessageMapContainer_Methods[] = { + {"__contains__", PyUpb_MapContainer_Contains, METH_O, + "Tests whether the map contains this element."}, + {"clear", PyUpb_MapContainer_Clear, METH_NOARGS, + "Removes all elements from the map."}, + {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS, + "Gets the value for the given key if present, or otherwise a default"}, + {"get_or_create", PyUpb_MapContainer_Subscript, METH_O, + "Alias for getitem, useful to make explicit that the map is mutated."}, + {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS, + "Return the class used to build Entries of (key, value) pairs."}, + {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O, + "Merges a map into the current map."}, + /* + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, + { "__reduce__", (PyCFunction)Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field." }, + */ + {NULL, NULL}, +}; + +static PyType_Slot PyUpb_MessageMapContainer_Slots[] = { + {Py_tp_dealloc, PyUpb_MapContainer_Dealloc}, + {Py_mp_length, PyUpb_MapContainer_Length}, + {Py_mp_subscript, PyUpb_MapContainer_Subscript}, + {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript}, + {Py_tp_methods, PyUpb_MessageMapContainer_Methods}, + {Py_tp_iter, PyUpb_MapIterator_New}, + {Py_tp_repr, PyUpb_MapContainer_Repr}, + {0, NULL}}; + +static PyType_Spec PyUpb_MessageMapContainer_Spec = { + PYUPB_MODULE_NAME ".MessageMapContainer", sizeof(PyUpb_MapContainer), 0, + Py_TPFLAGS_DEFAULT, PyUpb_MessageMapContainer_Slots}; + +// ----------------------------------------------------------------------------- +// MapIterator +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + PyUpb_MapContainer* map; // We own a reference. + size_t iter; + int version; +} PyUpb_MapIterator; + +static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyUpb_MapIterator* iter = + (void*)PyType_GenericAlloc(state->map_iterator_type, 0); + iter->map = map; + iter->iter = kUpb_Map_Begin; + iter->version = map->version; + Py_INCREF(map); + return &iter->ob_base; +} + +static void PyUpb_MapIterator_Dealloc(void* _self) { + PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self; + Py_DECREF(&self->map->ob_base); + PyUpb_Dealloc(_self); +} + +PyObject* PyUpb_MapIterator_IterNext(PyObject* _self) { + PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self; + if (self->version != self->map->version) { + return PyErr_Format(PyExc_RuntimeError, "Map modified during iteration."); + } + upb_Map* map = PyUpb_MapContainer_GetIfReified(self->map); + if (!map) return NULL; + upb_MessageValue key, val; + if (!upb_Map_Next(map, &key, &val, &self->iter)) return NULL; + const upb_FieldDef* f = PyUpb_MapContainer_GetField(self->map); + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0); + return PyUpb_UpbToPy(key, key_f, self->map->arena); +} + +static PyType_Slot PyUpb_MapIterator_Slots[] = { + {Py_tp_dealloc, PyUpb_MapIterator_Dealloc}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, PyUpb_MapIterator_IterNext}, + {0, NULL}}; + +static PyType_Spec PyUpb_MapIterator_Spec = { + PYUPB_MODULE_NAME ".MapIterator", sizeof(PyUpb_MapIterator), 0, + Py_TPFLAGS_DEFAULT, PyUpb_MapIterator_Slots}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +static PyObject* GetMutableMappingBase(void) { + PyObject* collections = NULL; + PyObject* mapping = NULL; + PyObject* bases = NULL; + if ((collections = PyImport_ImportModule("collections.abc")) && + (mapping = PyObject_GetAttrString(collections, "MutableMapping"))) { + bases = Py_BuildValue("(O)", mapping); + } + Py_XDECREF(collections); + Py_XDECREF(mapping); + return bases; +} + +bool PyUpb_Map_Init(PyObject* m) { + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + PyObject* bases = GetMutableMappingBase(); + if (!bases) return false; + + state->message_map_container_type = + PyUpb_AddClassWithBases(m, &PyUpb_MessageMapContainer_Spec, bases); + state->scalar_map_container_type = + PyUpb_AddClassWithBases(m, &PyUpb_ScalarMapContainer_Spec, bases); + state->map_iterator_type = PyUpb_AddClass(m, &PyUpb_MapIterator_Spec); + + Py_DECREF(bases); + + return state->message_map_container_type && + state->scalar_map_container_type && state->map_iterator_type; +}
diff --git a/python/map.h b/python/map.h new file mode 100644 index 0000000..6c2c47d --- /dev/null +++ b/python/map.h
@@ -0,0 +1,69 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_MAP_H__ +#define PYUPB_MAP_H__ + +#include <stdbool.h> + +#include "python/python_api.h" +#include "upb/reflection/def.h" + +// Creates a new repeated field stub for field `f` of message object `parent`. +// Precondition: `parent` must be a stub. +PyObject* PyUpb_MapContainer_NewStub(PyObject* parent, const upb_FieldDef* f, + PyObject* arena); + +// Returns a map object wrapping `map`, of field type `f`, which must be on +// `arena`. If an existing wrapper object exists, it will be returned, +// otherwise a new object will be created. The caller always owns a ref on the +// returned value. +PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_Map* map, + const upb_FieldDef* f, + PyObject* arena); + +// Reifies a map stub to point to the concrete data in `map`. +// If `map` is NULL, an appropriate empty map will be constructed. +void PyUpb_MapContainer_Reify(PyObject* self, upb_Map* map); + +// Reifies this map object if it is not already reified. +upb_Map* PyUpb_MapContainer_EnsureReified(PyObject* self); + +// Assigns `self[key] = val` for the map `self`. +int PyUpb_MapContainer_AssignSubscript(PyObject* self, PyObject* key, + PyObject* val); + +// Invalidates any existing iterators for the map `obj`. +void PyUpb_MapContainer_Invalidate(PyObject* obj); + +// Module-level init. +bool PyUpb_Map_Init(PyObject* m); + +#endif // PYUPB_MAP_H__
diff --git a/python/message.c b/python/message.c new file mode 100644 index 0000000..0777e7e --- /dev/null +++ b/python/message.c
@@ -0,0 +1,2019 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/message.h" + +#include "python/convert.h" +#include "python/descriptor.h" +#include "python/extension_dict.h" +#include "python/map.h" +#include "python/repeated.h" +#include "upb/message/copy.h" +#include "upb/reflection/def.h" +#include "upb/reflection/message.h" +#include "upb/text/encode.h" +#include "upb/util/required_fields.h" + +static const upb_MessageDef* PyUpb_MessageMeta_GetMsgdef(PyObject* cls); +static PyObject* PyUpb_MessageMeta_GetAttr(PyObject* self, PyObject* name); + +// ----------------------------------------------------------------------------- +// CPythonBits +// ----------------------------------------------------------------------------- + +// This struct contains a few things that are not exposed directly through the +// limited API, but that we can get at in somewhat more roundabout ways. The +// roundabout ways are slower, so we cache the values here. +// +// These values are valid to cache in a global, even across sub-interpreters, +// because they are not pointers to interpreter state. They are process +// globals that will be the same for any interpreter in this process. +typedef struct { + // For each member, we note the equivalent expression that we could use in the + // full (non-limited) API. + newfunc type_new; // PyTypeObject.tp_new + destructor type_dealloc; // PyTypeObject.tp_dealloc + getattrofunc type_getattro; // PyTypeObject.tp_getattro + setattrofunc type_setattro; // PyTypeObject.tp_setattro + size_t type_basicsize; // sizeof(PyHeapTypeObject) + traverseproc type_traverse; // PyTypeObject.tp_traverse + inquiry type_clear; // PyTypeObject.tp_clear + + // While we can refer to PY_VERSION_HEX in the limited API, this will give us + // the version of Python we were compiled against, which may be different + // than the version we are dynamically linked against. Here we want the + // version that is actually running in this process. + long python_version_hex; // PY_VERSION_HEX +} PyUpb_CPythonBits; + +// A global containing the values for this process. +PyUpb_CPythonBits cpython_bits; + +destructor upb_Pre310_PyType_GetDeallocSlot(PyTypeObject* type_subclass) { + // This is a bit desperate. We need type_dealloc(), but PyType_GetSlot(type, + // Py_tp_dealloc) will return subtype_dealloc(). There appears to be no way + // whatsoever to fetch type_dealloc() through the limited API until Python + // 3.10. + // + // To work around this so we attempt to find it by looking for the offset of + // tp_dealloc in PyTypeObject, then memcpy() it directly. This should always + // work in practice. + // + // Starting with Python 3.10 on you can call PyType_GetSlot() on non-heap + // types. We will be able to replace all this hack with just: + // + // PyType_GetSlot(&PyType_Type, Py_tp_dealloc) + // + destructor subtype_dealloc = PyType_GetSlot(type_subclass, Py_tp_dealloc); + for (size_t i = 0; i < 2000; i += sizeof(uintptr_t)) { + destructor maybe_subtype_dealloc; + memcpy(&maybe_subtype_dealloc, (char*)type_subclass + i, + sizeof(destructor)); + if (maybe_subtype_dealloc == subtype_dealloc) { + destructor type_dealloc; + memcpy(&type_dealloc, (char*)&PyType_Type + i, sizeof(destructor)); + return type_dealloc; + } + } + assert(false); + return NULL; +} + +static bool PyUpb_CPythonBits_Init(PyUpb_CPythonBits* bits) { + PyObject* bases = NULL; + PyTypeObject* type = NULL; + PyObject* size = NULL; + PyObject* sys = NULL; + PyObject* hex_version = NULL; + bool ret = false; + + // PyType_GetSlot() only works on heap types, so we cannot use it on + // &PyType_Type directly. Instead we create our own (temporary) type derived + // from PyType_Type: this will inherit all of the slots from PyType_Type, but + // as a heap type it can be queried with PyType_GetSlot(). + static PyType_Slot dummy_slots[] = {{0, NULL}}; + + static PyType_Spec dummy_spec = { + "module.DummyClass", // tp_name + 0, // To be filled in by size of base // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + dummy_slots, + }; + + bases = Py_BuildValue("(O)", &PyType_Type); + if (!bases) goto err; + type = (PyTypeObject*)PyType_FromSpecWithBases(&dummy_spec, bases); + if (!type) goto err; + + bits->type_new = PyType_GetSlot(type, Py_tp_new); + bits->type_dealloc = upb_Pre310_PyType_GetDeallocSlot(type); + bits->type_getattro = PyType_GetSlot(type, Py_tp_getattro); + bits->type_setattro = PyType_GetSlot(type, Py_tp_setattro); + bits->type_traverse = PyType_GetSlot(type, Py_tp_traverse); + bits->type_clear = PyType_GetSlot(type, Py_tp_clear); + + size = PyObject_GetAttrString((PyObject*)&PyType_Type, "__basicsize__"); + if (!size) goto err; + bits->type_basicsize = PyLong_AsLong(size); + if (bits->type_basicsize == -1) goto err; + + assert(bits->type_new); + assert(bits->type_dealloc); + assert(bits->type_getattro); + assert(bits->type_setattro); + assert(bits->type_traverse); + assert(bits->type_clear); + +#ifndef Py_LIMITED_API + assert(bits->type_new == PyType_Type.tp_new); + assert(bits->type_dealloc == PyType_Type.tp_dealloc); + assert(bits->type_getattro == PyType_Type.tp_getattro); + assert(bits->type_setattro == PyType_Type.tp_setattro); + assert(bits->type_basicsize == sizeof(PyHeapTypeObject)); + assert(bits->type_traverse == PyType_Type.tp_traverse); + assert(bits->type_clear == PyType_Type.tp_clear); +#endif + + sys = PyImport_ImportModule("sys"); + hex_version = PyObject_GetAttrString(sys, "hexversion"); + bits->python_version_hex = PyLong_AsLong(hex_version); + ret = true; + +err: + Py_XDECREF(bases); + Py_XDECREF(type); + Py_XDECREF(size); + Py_XDECREF(sys); + Py_XDECREF(hex_version); + return ret; +} + +// ----------------------------------------------------------------------------- +// Message +// ----------------------------------------------------------------------------- + +// The main message object. The type of the object (PyUpb_Message.ob_type) +// will be an instance of the PyUpb_MessageMeta type (defined below). So the +// chain is: +// FooMessage = MessageMeta(...) +// foo = FooMessage() +// +// Which becomes: +// Object C Struct Type Python type (ob_type) +// ----------------- ----------------- --------------------- +// foo PyUpb_Message FooMessage +// FooMessage PyUpb_MessageMeta message_meta_type +// message_meta_type PyTypeObject 'type' in Python +// +// A message object can be in one of two states: present or non-present. When +// a message is non-present, it stores a reference to its parent, and a write +// to any attribute will trigger the message to become present in its parent. +// The parent may also be non-present, in which case a mutation will trigger a +// chain reaction. +typedef struct PyUpb_Message { + PyObject_HEAD; + PyObject* arena; + uintptr_t def; // Tagged, low bit 1 == upb_FieldDef*, else upb_MessageDef* + union { + // when def is msgdef, the data for this msg. + upb_Message* msg; + // when def is fielddef, owning pointer to parent + struct PyUpb_Message* parent; + } ptr; + PyObject* ext_dict; // Weak pointer to extension dict, if any. + // name->obj dict for non-present msg/map/repeated, NULL if none. + PyUpb_WeakMap* unset_subobj_map; + int version; +} PyUpb_Message; + +static PyObject* PyUpb_Message_GetAttr(PyObject* _self, PyObject* attr); + +bool PyUpb_Message_IsStub(PyUpb_Message* msg) { return msg->def & 1; } + +const upb_FieldDef* PyUpb_Message_GetFieldDef(PyUpb_Message* msg) { + assert(PyUpb_Message_IsStub(msg)); + return (void*)(msg->def & ~(uintptr_t)1); +} + +static const upb_MessageDef* _PyUpb_Message_GetMsgdef(PyUpb_Message* msg) { + return PyUpb_Message_IsStub(msg) + ? upb_FieldDef_MessageSubDef(PyUpb_Message_GetFieldDef(msg)) + : (void*)msg->def; +} + +const upb_MessageDef* PyUpb_Message_GetMsgdef(PyObject* self) { + return _PyUpb_Message_GetMsgdef((PyUpb_Message*)self); +} + +static upb_Message* PyUpb_Message_GetMsg(PyUpb_Message* self) { + assert(!PyUpb_Message_IsStub(self)); + return self->ptr.msg; +} + +bool PyUpb_Message_TryCheck(PyObject* self) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyObject* type = (PyObject*)Py_TYPE(self); + return Py_TYPE(type) == state->message_meta_type; +} + +bool PyUpb_Message_Verify(PyObject* self) { + if (!PyUpb_Message_TryCheck(self)) { + PyErr_Format(PyExc_TypeError, "Expected a message object, but got %R.", + self); + return false; + } + return true; +} + +// If the message is reified, returns it. Otherwise, returns NULL. +// If NULL is returned, the object is empty and has no underlying data. +upb_Message* PyUpb_Message_GetIfReified(PyObject* _self) { + PyUpb_Message* self = (void*)_self; + return PyUpb_Message_IsStub(self) ? NULL : self->ptr.msg; +} + +static PyObject* PyUpb_Message_New(PyObject* cls, PyObject* unused_args, + PyObject* unused_kwargs) { + const upb_MessageDef* msgdef = PyUpb_MessageMeta_GetMsgdef(cls); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(msgdef); + PyUpb_Message* msg = (void*)PyType_GenericAlloc((PyTypeObject*)cls, 0); + msg->def = (uintptr_t)msgdef; + msg->arena = PyUpb_Arena_New(); + msg->ptr.msg = upb_Message_New(layout, PyUpb_Arena_Get(msg->arena)); + msg->unset_subobj_map = NULL; + msg->ext_dict = NULL; + msg->version = 0; + + PyObject* ret = &msg->ob_base; + PyUpb_ObjCache_Add(msg->ptr.msg, ret); + return ret; +} + +/* + * PyUpb_Message_LookupName() + * + * Tries to find a field or oneof named `py_name` in the message object `self`. + * The user must pass `f` and/or `o` to indicate whether a field or a oneof name + * is expected. If the name is found and it has an expected type, the function + * sets `*f` or `*o` respectively and returns true. Otherwise returns false + * and sets an exception of type `exc_type` if provided. + */ +static bool PyUpb_Message_LookupName(PyUpb_Message* self, PyObject* py_name, + const upb_FieldDef** f, + const upb_OneofDef** o, + PyObject* exc_type) { + assert(f || o); + Py_ssize_t size; + const char* name = NULL; + if (PyUnicode_Check(py_name)) { + name = PyUnicode_AsUTF8AndSize(py_name, &size); + } else if (PyBytes_Check(py_name)) { + PyBytes_AsStringAndSize(py_name, (char**)&name, &size); + } + if (!name) { + PyErr_Format(exc_type, + "Expected a field name, but got non-string argument %S.", + py_name); + return false; + } + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + + if (!upb_MessageDef_FindByNameWithSize(msgdef, name, size, f, o)) { + if (exc_type) { + PyErr_Format(exc_type, "Protocol message %s has no \"%s\" field.", + upb_MessageDef_Name(msgdef), name); + } + return false; + } + + if (!o && !*f) { + if (exc_type) { + PyErr_Format(exc_type, "Expected a field name, but got oneof name %s.", + name); + } + return false; + } + + if (!f && !*o) { + if (exc_type) { + PyErr_Format(exc_type, "Expected a oneof name, but got field name %s.", + name); + } + return false; + } + + return true; +} + +static bool PyUpb_Message_InitMessageMapEntry(PyObject* dst, PyObject* src) { + if (!src || !dst) return false; + + PyObject* ok = PyObject_CallMethod(dst, "CopyFrom", "O", src); + if (!ok) return false; + Py_DECREF(ok); + + return true; +} + +int PyUpb_Message_InitMapAttributes(PyObject* map, PyObject* value, + const upb_FieldDef* f) { + const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f); + const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1); + PyObject* it = NULL; + PyObject* tmp = NULL; + int ret = -1; + if (upb_FieldDef_IsSubMessage(val_f)) { + it = PyObject_GetIter(value); + if (it == NULL) { + PyErr_Format(PyExc_TypeError, "Argument for field %s is not iterable", + upb_FieldDef_FullName(f)); + goto err; + } + PyObject* e; + while ((e = PyIter_Next(it)) != NULL) { + PyObject* src = PyObject_GetItem(value, e); + PyObject* dst = PyObject_GetItem(map, e); + Py_DECREF(e); + bool ok = PyUpb_Message_InitMessageMapEntry(dst, src); + Py_XDECREF(src); + Py_XDECREF(dst); + if (!ok) goto err; + } + } else { + tmp = PyObject_CallMethod(map, "update", "O", value); + if (!tmp) goto err; + } + ret = 0; + +err: + Py_XDECREF(it); + Py_XDECREF(tmp); + return ret; +} + +void PyUpb_Message_EnsureReified(PyUpb_Message* self); + +static bool PyUpb_Message_InitMapAttribute(PyObject* _self, PyObject* name, + const upb_FieldDef* f, + PyObject* value) { + PyObject* map = PyUpb_Message_GetAttr(_self, name); + int ok = PyUpb_Message_InitMapAttributes(map, value, f); + Py_DECREF(map); + return ok >= 0; +} + +static bool PyUpb_Message_InitRepeatedMessageAttribute(PyObject* _self, + PyObject* repeated, + PyObject* value, + const upb_FieldDef* f) { + PyObject* it = PyObject_GetIter(value); + if (!it) { + PyErr_Format(PyExc_TypeError, "Argument for field %s is not iterable", + upb_FieldDef_FullName(f)); + return false; + } + PyObject* e = NULL; + PyObject* m = NULL; + while ((e = PyIter_Next(it)) != NULL) { + if (PyDict_Check(e)) { + m = PyUpb_RepeatedCompositeContainer_Add(repeated, NULL, e); + if (!m) goto err; + } else { + m = PyUpb_RepeatedCompositeContainer_Add(repeated, NULL, NULL); + if (!m) goto err; + PyObject* merged = PyUpb_Message_MergeFrom(m, e); + if (!merged) goto err; + Py_DECREF(merged); + } + Py_DECREF(e); + Py_DECREF(m); + m = NULL; + } + +err: + Py_XDECREF(it); + Py_XDECREF(e); + Py_XDECREF(m); + return !PyErr_Occurred(); // Check PyIter_Next() exit. +} + +static bool PyUpb_Message_InitRepeatedAttribute(PyObject* _self, PyObject* name, + PyObject* value) { + PyUpb_Message* self = (void*)_self; + const upb_FieldDef* field; + if (!PyUpb_Message_LookupName(self, name, &field, NULL, + PyExc_AttributeError)) { + return false; + } + bool ok = false; + PyObject* repeated = PyUpb_Message_GetFieldValue(_self, field); + PyObject* tmp = NULL; + if (!repeated) goto err; + if (upb_FieldDef_IsSubMessage(field)) { + if (!PyUpb_Message_InitRepeatedMessageAttribute(_self, repeated, value, + field)) { + goto err; + } + } else { + tmp = PyUpb_RepeatedContainer_Extend(repeated, value); + if (!tmp) goto err; + } + ok = true; + +err: + Py_XDECREF(repeated); + Py_XDECREF(tmp); + return ok; +} + +static bool PyUpb_Message_InitMessageAttribute(PyObject* _self, PyObject* name, + PyObject* value) { + PyObject* submsg = PyUpb_Message_GetAttr(_self, name); + if (!submsg) return -1; + assert(!PyErr_Occurred()); + bool ok; + if (PyUpb_Message_TryCheck(value)) { + PyObject* tmp = PyUpb_Message_MergeFrom(submsg, value); + ok = tmp != NULL; + Py_XDECREF(tmp); + } else if (PyDict_Check(value)) { + assert(!PyErr_Occurred()); + ok = PyUpb_Message_InitAttributes(submsg, NULL, value) >= 0; + } else { + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(_self); + PyErr_Format(PyExc_TypeError, "Message must be initialized with a dict: %s", + upb_MessageDef_FullName(m)); + ok = false; + } + Py_DECREF(submsg); + return ok; +} + +static bool PyUpb_Message_InitScalarAttribute(upb_Message* msg, + const upb_FieldDef* f, + PyObject* value, + upb_Arena* arena) { + upb_MessageValue msgval; + assert(!PyErr_Occurred()); + if (!PyUpb_PyToUpb(value, f, &msgval, arena)) return false; + upb_Message_SetFieldByDef(msg, f, msgval, arena); + return true; +} + +int PyUpb_Message_InitAttributes(PyObject* _self, PyObject* args, + PyObject* kwargs) { + assert(!PyErr_Occurred()); + + if (args != NULL && PyTuple_Size(args) != 0) { + PyErr_SetString(PyExc_TypeError, "No positional arguments allowed"); + return -1; + } + + if (kwargs == NULL) return 0; + + PyUpb_Message* self = (void*)_self; + Py_ssize_t pos = 0; + PyObject* name; + PyObject* value; + PyUpb_Message_EnsureReified(self); + upb_Message* msg = PyUpb_Message_GetMsg(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + + while (PyDict_Next(kwargs, &pos, &name, &value)) { + assert(!PyErr_Occurred()); + const upb_FieldDef* f; + assert(!PyErr_Occurred()); + if (!PyUpb_Message_LookupName(self, name, &f, NULL, PyExc_ValueError)) { + return -1; + } + + if (value == Py_None) continue; // Ignored. + + assert(!PyErr_Occurred()); + + if (upb_FieldDef_IsMap(f)) { + if (!PyUpb_Message_InitMapAttribute(_self, name, f, value)) return -1; + } else if (upb_FieldDef_IsRepeated(f)) { + if (!PyUpb_Message_InitRepeatedAttribute(_self, name, value)) return -1; + } else if (upb_FieldDef_IsSubMessage(f)) { + if (!PyUpb_Message_InitMessageAttribute(_self, name, value)) return -1; + } else { + if (!PyUpb_Message_InitScalarAttribute(msg, f, value, arena)) return -1; + } + if (PyErr_Occurred()) return -1; + } + + if (PyErr_Occurred()) return -1; + return 0; +} + +static int PyUpb_Message_Init(PyObject* _self, PyObject* args, + PyObject* kwargs) { + if (args != NULL && PyTuple_Size(args) != 0) { + PyErr_SetString(PyExc_TypeError, "No positional arguments allowed"); + return -1; + } + + return PyUpb_Message_InitAttributes(_self, args, kwargs); +} + +static PyObject* PyUpb_Message_NewStub(PyObject* parent, const upb_FieldDef* f, + PyObject* arena) { + const upb_MessageDef* sub_m = upb_FieldDef_MessageSubDef(f); + PyObject* cls = PyUpb_Descriptor_GetClass(sub_m); + + PyUpb_Message* msg = (void*)PyType_GenericAlloc((PyTypeObject*)cls, 0); + msg->def = (uintptr_t)f | 1; + msg->arena = arena; + msg->ptr.parent = (PyUpb_Message*)parent; + msg->unset_subobj_map = NULL; + msg->ext_dict = NULL; + msg->version = 0; + + Py_DECREF(cls); + Py_INCREF(parent); + Py_INCREF(arena); + return &msg->ob_base; +} + +static bool PyUpb_Message_IsEmpty(const upb_Message* msg, + const upb_MessageDef* m, + const upb_DefPool* ext_pool) { + if (!msg) return true; + + size_t iter = kUpb_Message_Begin; + const upb_FieldDef* f; + upb_MessageValue val; + if (upb_Message_Next(msg, m, ext_pool, &f, &val, &iter)) return false; + + size_t len; + (void)upb_Message_GetUnknown(msg, &len); + return len == 0; +} + +static bool PyUpb_Message_IsEqual(PyUpb_Message* m1, PyObject* _m2) { + PyUpb_Message* m2 = (void*)_m2; + if (m1 == m2) return true; + if (!PyObject_TypeCheck(_m2, m1->ob_base.ob_type)) { + return false; + } + const upb_MessageDef* m1_msgdef = _PyUpb_Message_GetMsgdef(m1); +#ifndef NDEBUG + const upb_MessageDef* m2_msgdef = _PyUpb_Message_GetMsgdef(m2); + assert(m1_msgdef == m2_msgdef); +#endif + const upb_Message* m1_msg = PyUpb_Message_GetIfReified((PyObject*)m1); + const upb_Message* m2_msg = PyUpb_Message_GetIfReified(_m2); + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m1_msgdef)); + + const bool e1 = PyUpb_Message_IsEmpty(m1_msg, m1_msgdef, symtab); + const bool e2 = PyUpb_Message_IsEmpty(m2_msg, m1_msgdef, symtab); + if (e1 || e2) return e1 && e2; + + return upb_Message_IsEqual(m1_msg, m2_msg, m1_msgdef); +} + +static const upb_FieldDef* PyUpb_Message_InitAsMsg(PyUpb_Message* m, + upb_Arena* arena) { + const upb_FieldDef* f = PyUpb_Message_GetFieldDef(m); + const upb_MessageDef* m2 = upb_FieldDef_MessageSubDef(f); + m->ptr.msg = upb_Message_New(upb_MessageDef_MiniTable(m2), arena); + m->def = (uintptr_t)m2; + PyUpb_ObjCache_Add(m->ptr.msg, &m->ob_base); + return f; +} + +static void PyUpb_Message_SetField(PyUpb_Message* parent, const upb_FieldDef* f, + PyUpb_Message* child, upb_Arena* arena) { + upb_MessageValue msgval = {.msg_val = PyUpb_Message_GetMsg(child)}; + upb_Message_SetFieldByDef(PyUpb_Message_GetMsg(parent), f, msgval, arena); + PyUpb_WeakMap_Delete(parent->unset_subobj_map, f); + // Releases a ref previously owned by child->ptr.parent of our child. + Py_DECREF(child); +} + +/* + * PyUpb_Message_EnsureReified() + * + * This implements the "expando" behavior of Python protos: + * foo = FooProto() + * + * # The intermediate messages don't really exist, and won't be serialized. + * x = foo.bar.bar.bar.bar.bar.baz + * + * # Now all the intermediate objects are created. + * foo.bar.bar.bar.bar.bar.baz = 5 + * + * This function should be called before performing any mutation of a protobuf + * object. + * + * Post-condition: + * PyUpb_Message_IsStub(self) is false + */ +void PyUpb_Message_EnsureReified(PyUpb_Message* self) { + if (!PyUpb_Message_IsStub(self)) return; + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + + // This is a non-present message. We need to create a real upb_Message for + // this object and every parent until we reach a present message. + PyUpb_Message* child = self; + PyUpb_Message* parent = self->ptr.parent; + const upb_FieldDef* child_f = PyUpb_Message_InitAsMsg(child, arena); + Py_INCREF(child); // To avoid a special-case in PyUpb_Message_SetField(). + + do { + PyUpb_Message* next_parent = parent->ptr.parent; + const upb_FieldDef* parent_f = NULL; + if (PyUpb_Message_IsStub(parent)) { + parent_f = PyUpb_Message_InitAsMsg(parent, arena); + } + PyUpb_Message_SetField(parent, child_f, child, arena); + child = parent; + child_f = parent_f; + parent = next_parent; + } while (child_f); + + // Releases ref previously owned by child->ptr.parent of our child. + Py_DECREF(child); + self->version++; +} + +static void PyUpb_Message_SyncSubobjs(PyUpb_Message* self); + +/* + * PyUpb_Message_Reify() + * + * The message equivalent of PyUpb_*Container_Reify(), this transitions + * the wrapper from the unset state (owning a reference on self->ptr.parent) to + * the set state (having a non-owning pointer to self->ptr.msg). + */ +static void PyUpb_Message_Reify(PyUpb_Message* self, const upb_FieldDef* f, + upb_Message* msg) { + assert(f == PyUpb_Message_GetFieldDef(self)); + if (!msg) { + const upb_MessageDef* msgdef = PyUpb_Message_GetMsgdef((PyObject*)self); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(msgdef); + msg = upb_Message_New(layout, PyUpb_Arena_Get(self->arena)); + } + PyUpb_ObjCache_Add(msg, &self->ob_base); + Py_DECREF(&self->ptr.parent->ob_base); + self->ptr.msg = msg; // Overwrites self->ptr.parent + self->def = (uintptr_t)upb_FieldDef_MessageSubDef(f); + PyUpb_Message_SyncSubobjs(self); +} + +/* + * PyUpb_Message_SyncSubobjs() + * + * This operation must be invoked whenever the underlying upb_Message has been + * mutated directly in C. This will attach any newly-present field data + * to previously returned stub wrapper objects. + * + * For example: + * foo = FooMessage() + * sub = foo.submsg # Empty, unset sub-message + * + * # SyncSubobjs() is required to connect our existing 'sub' wrapper to the + * # newly created foo.submsg data in C. + * foo.MergeFrom(FooMessage(submsg={})) + * + * This requires that all of the new sub-objects that have appeared are owned + * by `self`'s arena. + */ +static void PyUpb_Message_SyncSubobjs(PyUpb_Message* self) { + PyUpb_WeakMap* subobj_map = self->unset_subobj_map; + if (!subobj_map) return; + + upb_Message* msg = PyUpb_Message_GetMsg(self); + intptr_t iter = PYUPB_WEAKMAP_BEGIN; + const void* key; + PyObject* obj; + + // The last ref to this message could disappear during iteration. + // When we call PyUpb_*Container_Reify() below, the container will drop + // its ref on `self`. If that was the last ref on self, the object will be + // deleted, and `subobj_map` along with it. We need it to live until we are + // done iterating. + Py_INCREF(&self->ob_base); + + while (PyUpb_WeakMap_Next(subobj_map, &key, &obj, &iter)) { + const upb_FieldDef* f = key; + if (upb_FieldDef_HasPresence(f) && !upb_Message_HasFieldByDef(msg, f)) + continue; + upb_MessageValue msgval = upb_Message_GetFieldByDef(msg, f); + PyUpb_WeakMap_DeleteIter(subobj_map, &iter); + if (upb_FieldDef_IsMap(f)) { + if (!msgval.map_val) continue; + PyUpb_MapContainer_Reify(obj, (upb_Map*)msgval.map_val); + } else if (upb_FieldDef_IsRepeated(f)) { + if (!msgval.array_val) continue; + PyUpb_RepeatedContainer_Reify(obj, (upb_Array*)msgval.array_val); + } else { + PyUpb_Message* sub = (void*)obj; + assert(self == sub->ptr.parent); + PyUpb_Message_Reify(sub, f, (upb_Message*)msgval.msg_val); + } + } + + Py_DECREF(&self->ob_base); + + // TODO: present fields need to be iterated too if they can reach + // a WeakMap. +} + +static PyObject* PyUpb_Message_ToString(PyUpb_Message* self) { + if (PyUpb_Message_IsStub(self)) { + return PyUnicode_FromStringAndSize(NULL, 0); + } + upb_Message* msg = PyUpb_Message_GetMsg(self); + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(msgdef)); + char buf[1024]; + int options = UPB_TXTENC_SKIPUNKNOWN; + size_t size = upb_TextEncode(msg, msgdef, symtab, options, buf, sizeof(buf)); + if (size < sizeof(buf)) { + return PyUnicode_FromStringAndSize(buf, size); + } else { + char* buf2 = malloc(size + 1); + size_t size2 = upb_TextEncode(msg, msgdef, symtab, options, buf2, size + 1); + assert(size == size2); + PyObject* ret = PyUnicode_FromStringAndSize(buf2, size2); + free(buf2); + return ret; + } +} + +static PyObject* PyUpb_Message_RichCompare(PyObject* _self, PyObject* other, + int opid) { + PyUpb_Message* self = (void*)_self; + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + if (!PyObject_TypeCheck(other, Py_TYPE(self))) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + bool ret = PyUpb_Message_IsEqual(self, other); + if (opid == Py_NE) ret = !ret; + return PyBool_FromLong(ret); +} + +void PyUpb_Message_CacheDelete(PyObject* _self, const upb_FieldDef* f) { + PyUpb_Message* self = (void*)_self; + PyUpb_WeakMap_Delete(self->unset_subobj_map, f); +} + +void PyUpb_Message_SetConcreteSubobj(PyObject* _self, const upb_FieldDef* f, + upb_MessageValue subobj) { + PyUpb_Message* self = (void*)_self; + PyUpb_Message_EnsureReified(self); + PyUpb_Message_CacheDelete(_self, f); + upb_Message_SetFieldByDef(self->ptr.msg, f, subobj, + PyUpb_Arena_Get(self->arena)); +} + +static void PyUpb_Message_Dealloc(PyObject* _self) { + PyUpb_Message* self = (void*)_self; + + if (PyUpb_Message_IsStub(self)) { + PyUpb_Message_CacheDelete((PyObject*)self->ptr.parent, + PyUpb_Message_GetFieldDef(self)); + Py_DECREF(self->ptr.parent); + } else { + PyUpb_ObjCache_Delete(self->ptr.msg); + } + + if (self->unset_subobj_map) { + PyUpb_WeakMap_Free(self->unset_subobj_map); + } + + Py_DECREF(self->arena); + + // We do not use PyUpb_Dealloc() here because Message is a base type and for + // base types there is a bug we have to work around in this case (see below). + PyTypeObject* tp = Py_TYPE(self); + freefunc tp_free = PyType_GetSlot(tp, Py_tp_free); + tp_free(self); + + if (cpython_bits.python_version_hex >= 0x03080000) { + // Prior to Python 3.8 there is a bug where deallocating the type here would + // lead to a double-decref: https://bugs.python.org/issue37879 + Py_DECREF(tp); + } +} + +PyObject* PyUpb_Message_Get(upb_Message* u_msg, const upb_MessageDef* m, + PyObject* arena) { + PyObject* ret = PyUpb_ObjCache_Get(u_msg); + if (ret) return ret; + + PyObject* cls = PyUpb_Descriptor_GetClass(m); + // It is not safe to use PyObject_{,GC}_New() due to: + // https://bugs.python.org/issue35810 + PyUpb_Message* py_msg = (void*)PyType_GenericAlloc((PyTypeObject*)cls, 0); + py_msg->arena = arena; + py_msg->def = (uintptr_t)m; + py_msg->ptr.msg = u_msg; + py_msg->unset_subobj_map = NULL; + py_msg->ext_dict = NULL; + py_msg->version = 0; + ret = &py_msg->ob_base; + Py_DECREF(cls); + Py_INCREF(arena); + PyUpb_ObjCache_Add(u_msg, ret); + return ret; +} + +/* PyUpb_Message_GetStub() + * + * Non-present messages return "stub" objects that point to their parent, but + * will materialize into real upb objects if they are mutated. + * + * Note: we do *not* create stubs for repeated/map fields unless the parent + * is a stub: + * + * msg = TestMessage() + * msg.submessage # (A) Creates a stub + * msg.repeated_foo # (B) Does *not* create a stub + * msg.submessage.repeated_bar # (C) Creates a stub + * + * In case (B) we have some freedom: we could either create a stub, or create + * a reified object with underlying data. It appears that either could work + * equally well, with no observable change to users. There isn't a clear + * advantage to either choice. We choose to follow the behavior of the + * pre-existing C++ behavior for consistency, but if it becomes apparent that + * there would be some benefit to reversing this decision, it should be totally + * within the realm of possibility. + */ +PyObject* PyUpb_Message_GetStub(PyUpb_Message* self, + const upb_FieldDef* field) { + PyObject* _self = (void*)self; + if (!self->unset_subobj_map) { + self->unset_subobj_map = PyUpb_WeakMap_New(); + } + PyObject* subobj = PyUpb_WeakMap_Get(self->unset_subobj_map, field); + + if (subobj) return subobj; + + if (upb_FieldDef_IsMap(field)) { + subobj = PyUpb_MapContainer_NewStub(_self, field, self->arena); + } else if (upb_FieldDef_IsRepeated(field)) { + subobj = PyUpb_RepeatedContainer_NewStub(_self, field, self->arena); + } else { + subobj = PyUpb_Message_NewStub(&self->ob_base, field, self->arena); + } + PyUpb_WeakMap_Add(self->unset_subobj_map, field, subobj); + + assert(!PyErr_Occurred()); + return subobj; +} + +PyObject* PyUpb_Message_GetPresentWrapper(PyUpb_Message* self, + const upb_FieldDef* field) { + assert(!PyUpb_Message_IsStub(self)); + upb_MutableMessageValue mutval = + upb_Message_Mutable(self->ptr.msg, field, PyUpb_Arena_Get(self->arena)); + if (upb_FieldDef_IsMap(field)) { + return PyUpb_MapContainer_GetOrCreateWrapper(mutval.map, field, + self->arena); + } else { + return PyUpb_RepeatedContainer_GetOrCreateWrapper(mutval.array, field, + self->arena); + } +} + +PyObject* PyUpb_Message_GetScalarValue(PyUpb_Message* self, + const upb_FieldDef* field) { + upb_MessageValue val; + if (PyUpb_Message_IsStub(self)) { + // Unset message always returns default values. + val = upb_FieldDef_Default(field); + } else { + val = upb_Message_GetFieldByDef(self->ptr.msg, field); + } + return PyUpb_UpbToPy(val, field, self->arena); +} + +/* + * PyUpb_Message_GetFieldValue() + * + * Implements the equivalent of getattr(msg, field), once `field` has + * already been resolved to a `upb_FieldDef*`. + * + * This may involve constructing a wrapper object for the given field, or + * returning one that was previously constructed. If the field is not actually + * set, the wrapper object will be an "unset" object that is not actually + * connected to any C data. + */ +PyObject* PyUpb_Message_GetFieldValue(PyObject* _self, + const upb_FieldDef* field) { + PyUpb_Message* self = (void*)_self; + assert(upb_FieldDef_ContainingType(field) == PyUpb_Message_GetMsgdef(_self)); + bool submsg = upb_FieldDef_IsSubMessage(field); + bool seq = upb_FieldDef_IsRepeated(field); + + if ((PyUpb_Message_IsStub(self) && (submsg || seq)) || + (submsg && !seq && !upb_Message_HasFieldByDef(self->ptr.msg, field))) { + return PyUpb_Message_GetStub(self, field); + } else if (seq) { + return PyUpb_Message_GetPresentWrapper(self, field); + } else { + return PyUpb_Message_GetScalarValue(self, field); + } +} + +int PyUpb_Message_SetFieldValue(PyObject* _self, const upb_FieldDef* field, + PyObject* value, PyObject* exc) { + PyUpb_Message* self = (void*)_self; + assert(value); + + if (upb_FieldDef_IsSubMessage(field) || upb_FieldDef_IsRepeated(field)) { + PyErr_Format(exc, + "Assignment not allowed to message, map, or repeated " + "field \"%s\" in protocol message object.", + upb_FieldDef_Name(field)); + return -1; + } + + PyUpb_Message_EnsureReified(self); + + upb_MessageValue val; + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + if (!PyUpb_PyToUpb(value, field, &val, arena)) { + return -1; + } + + upb_Message_SetFieldByDef(self->ptr.msg, field, val, arena); + return 0; +} + +int PyUpb_Message_GetVersion(PyObject* _self) { + PyUpb_Message* self = (void*)_self; + return self->version; +} + +/* + * PyUpb_Message_GetAttr() + * + * Implements: + * foo = msg.foo + * + * Attribute lookup must find both message fields and base class methods like + * msg.SerializeToString(). + */ +__attribute__((flatten)) static PyObject* PyUpb_Message_GetAttr( + PyObject* _self, PyObject* attr) { + PyUpb_Message* self = (void*)_self; + + // Lookup field by name. + const upb_FieldDef* field; + if (PyUpb_Message_LookupName(self, attr, &field, NULL, NULL)) { + return PyUpb_Message_GetFieldValue(_self, field); + } + + // Check base class attributes. + assert(!PyErr_Occurred()); + PyObject* ret = PyObject_GenericGetAttr(_self, attr); + if (ret) return ret; + + // Swallow AttributeError if it occurred and try again on the metaclass + // to pick up class attributes. But we have to special-case "Extensions" + // which affirmatively returns AttributeError when a message is not + // extendable. + const char* name; + if (PyErr_ExceptionMatches(PyExc_AttributeError) && + (name = PyUpb_GetStrData(attr)) && strcmp(name, "Extensions") != 0) { + PyErr_Clear(); + return PyUpb_MessageMeta_GetAttr((PyObject*)Py_TYPE(_self), attr); + } + + return NULL; +} + +/* + * PyUpb_Message_SetAttr() + * + * Implements: + * msg.foo = foo + */ +static int PyUpb_Message_SetAttr(PyObject* _self, PyObject* attr, + PyObject* value) { + PyUpb_Message* self = (void*)_self; + const upb_FieldDef* field; + if (!PyUpb_Message_LookupName(self, attr, &field, NULL, + PyExc_AttributeError)) { + return -1; + } + + return PyUpb_Message_SetFieldValue(_self, field, value, PyExc_AttributeError); +} + +static PyObject* PyUpb_Message_HasField(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + const upb_FieldDef* field; + const upb_OneofDef* oneof; + + if (!PyUpb_Message_LookupName(self, arg, &field, &oneof, PyExc_ValueError)) { + return NULL; + } + + if (field && !upb_FieldDef_HasPresence(field)) { + PyErr_Format(PyExc_ValueError, "Field %s does not have presence.", + upb_FieldDef_FullName(field)); + return NULL; + } + + if (PyUpb_Message_IsStub(self)) Py_RETURN_FALSE; + + return PyBool_FromLong(field ? upb_Message_HasFieldByDef(self->ptr.msg, field) + : upb_Message_WhichOneof(self->ptr.msg, oneof) != + NULL); +} + +static PyObject* PyUpb_Message_FindInitializationErrors(PyObject* _self, + PyObject* arg); + +static PyObject* PyUpb_Message_IsInitializedAppendErrors(PyObject* _self, + PyObject* errors) { + PyObject* list = PyUpb_Message_FindInitializationErrors(_self, NULL); + if (!list) return NULL; + bool ok = PyList_Size(list) == 0; + PyObject* ret = NULL; + PyObject* extend_result = NULL; + if (!ok) { + extend_result = PyObject_CallMethod(errors, "extend", "O", list); + if (!extend_result) goto done; + } + ret = PyBool_FromLong(ok); + +done: + Py_XDECREF(list); + Py_XDECREF(extend_result); + return ret; +} + +static PyObject* PyUpb_Message_IsInitialized(PyObject* _self, PyObject* args) { + PyObject* errors = NULL; + if (!PyArg_ParseTuple(args, "|O", &errors)) { + return NULL; + } + if (errors) { + // We need to collect a list of unset required fields and append it to + // `errors`. + return PyUpb_Message_IsInitializedAppendErrors(_self, errors); + } else { + // We just need to return a boolean "true" or "false" for whether all + // required fields are set. + upb_Message* msg = PyUpb_Message_GetIfReified(_self); + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(_self); + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m)); + bool initialized = !upb_util_HasUnsetRequired(msg, m, symtab, NULL); + return PyBool_FromLong(initialized); + } +} + +static PyObject* PyUpb_Message_ListFieldsItemKey(PyObject* self, + PyObject* val) { + assert(PyTuple_Check(val)); + PyObject* field = PyTuple_GetItem(val, 0); + const upb_FieldDef* f = PyUpb_FieldDescriptor_GetDef(field); + return PyLong_FromLong(upb_FieldDef_Number(f)); +} + +static PyObject* PyUpb_Message_CheckCalledFromGeneratedFile( + PyObject* unused, PyObject* unused_arg) { + PyErr_SetString( + PyExc_TypeError, + "Descriptors cannot be created directly.\n" + "If this call came from a _pb2.py file, your generated code is out of " + "date and must be regenerated with protoc >= 3.19.0.\n" + "If you cannot immediately regenerate your protos, some other possible " + "workarounds are:\n" + " 1. Downgrade the protobuf package to 3.20.x or lower.\n" + " 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will " + "use pure-Python parsing and will be much slower).\n" + "\n" + "More information: " + "https://developers.google.com/protocol-buffers/docs/news/" + "2022-05-06#python-updates"); + return NULL; +} + +static bool PyUpb_Message_SortFieldList(PyObject* list) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + bool ok = false; + PyObject* args = PyTuple_New(0); + PyObject* kwargs = PyDict_New(); + PyObject* method = PyObject_GetAttrString(list, "sort"); + PyObject* call_result = NULL; + if (!args || !kwargs || !method) goto err; + if (PyDict_SetItemString(kwargs, "key", state->listfields_item_key) < 0) { + goto err; + } + call_result = PyObject_Call(method, args, kwargs); + if (!call_result) goto err; + ok = true; + +err: + Py_XDECREF(method); + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(call_result); + return ok; +} + +static PyObject* PyUpb_Message_ListFields(PyObject* _self, PyObject* arg) { + PyObject* list = PyList_New(0); + upb_Message* msg = PyUpb_Message_GetIfReified(_self); + if (!msg) return list; + + size_t iter1 = kUpb_Message_Begin; + const upb_MessageDef* m = PyUpb_Message_GetMsgdef(_self); + const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m)); + const upb_FieldDef* f; + PyObject* field_desc = NULL; + PyObject* py_val = NULL; + PyObject* tuple = NULL; + upb_MessageValue val; + uint32_t last_field = 0; + bool in_order = true; + while (upb_Message_Next(msg, m, symtab, &f, &val, &iter1)) { + const uint32_t field_number = upb_FieldDef_Number(f); + if (field_number < last_field) in_order = false; + last_field = field_number; + PyObject* field_desc = PyUpb_FieldDescriptor_Get(f); + PyObject* py_val = PyUpb_Message_GetFieldValue(_self, f); + if (!field_desc || !py_val) goto err; + PyObject* tuple = Py_BuildValue("(NN)", field_desc, py_val); + field_desc = NULL; + py_val = NULL; + if (!tuple) goto err; + if (PyList_Append(list, tuple)) goto err; + Py_DECREF(tuple); + tuple = NULL; + } + + // Users rely on fields being returned in field number order. + if (!in_order && !PyUpb_Message_SortFieldList(list)) goto err; + + return list; + +err: + Py_XDECREF(field_desc); + Py_XDECREF(py_val); + Py_XDECREF(tuple); + Py_DECREF(list); + return NULL; +} + +PyObject* PyUpb_Message_MergeFrom(PyObject* self, PyObject* arg) { + if (self->ob_type != arg->ob_type) { + PyErr_Format(PyExc_TypeError, + "Parameter to MergeFrom() must be instance of same class: " + "expected %S got %S.", + Py_TYPE(self), Py_TYPE(arg)); + return NULL; + } + // OPT: exit if src is empty. + PyObject* subargs = PyTuple_New(0); + PyObject* serialized = + PyUpb_Message_SerializePartialToString(arg, subargs, NULL); + Py_DECREF(subargs); + if (!serialized) return NULL; + PyObject* ret = PyUpb_Message_MergeFromString(self, serialized); + Py_DECREF(serialized); + Py_XDECREF(ret); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_Clear(PyUpb_Message* self); + +static PyObject* PyUpb_Message_CopyFrom(PyObject* _self, PyObject* arg) { + if (_self->ob_type != arg->ob_type) { + PyErr_Format(PyExc_TypeError, + "Parameter to CopyFrom() must be instance of same class: " + "expected %S got %S.", + Py_TYPE(_self), Py_TYPE(arg)); + return NULL; + } + if (_self == arg) { + Py_RETURN_NONE; + } + PyUpb_Message* self = (void*)_self; + PyUpb_Message* other = (void*)arg; + PyUpb_Message_EnsureReified(self); + + const upb_Message* other_msg = PyUpb_Message_GetIfReified((PyObject*)other); + if (other_msg) { + upb_Message_DeepCopy( + self->ptr.msg, other_msg, + upb_MessageDef_MiniTable((const upb_MessageDef*)other->def), + PyUpb_Arena_Get(self->arena)); + } else { + PyObject* tmp = PyUpb_Message_Clear(self); + Py_DECREF(tmp); + } + PyUpb_Message_SyncSubobjs(self); + + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_SetInParent(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + PyUpb_Message_EnsureReified(self); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_UnknownFields(PyObject* _self, PyObject* arg) { + // TODO: re-enable when unknown fields are added. + // return PyUpb_UnknownFields_New(_self); + PyErr_SetString(PyExc_NotImplementedError, "unknown field accessor"); + return NULL; +} + +PyObject* PyUpb_Message_MergeFromString(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + char* buf; + Py_ssize_t size; + PyObject* bytes = NULL; + + if (PyMemoryView_Check(arg)) { + bytes = PyBytes_FromObject(arg); + // Cannot fail when passed something of the correct type. + int err = PyBytes_AsStringAndSize(bytes, &buf, &size); + (void)err; + assert(err >= 0); + } else if (PyBytes_AsStringAndSize(arg, &buf, &size) < 0) { + return NULL; + } + + PyUpb_Message_EnsureReified(self); + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + const upb_FileDef* file = upb_MessageDef_File(msgdef); + const upb_ExtensionRegistry* extreg = + upb_DefPool_ExtensionRegistry(upb_FileDef_Pool(file)); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(msgdef); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + int options = + upb_DecodeOptions_MaxDepth(state->allow_oversize_protos ? UINT16_MAX : 0); + upb_DecodeStatus status = + upb_Decode(buf, size, self->ptr.msg, layout, extreg, options, arena); + Py_XDECREF(bytes); + if (status != kUpb_DecodeStatus_Ok) { + PyErr_Format(state->decode_error_class, "Error parsing message"); + return NULL; + } + PyUpb_Message_SyncSubobjs(self); + return PyLong_FromSsize_t(size); +} + +static PyObject* PyUpb_Message_ParseFromString(PyObject* self, PyObject* arg) { + PyObject* tmp = PyUpb_Message_Clear((PyUpb_Message*)self); + Py_DECREF(tmp); + return PyUpb_Message_MergeFromString(self, arg); +} + +static PyObject* PyUpb_Message_ByteSize(PyObject* self, PyObject* args) { + // TODO: At the + // moment upb does not have a "byte size" function, so we just serialize to + // string and get the size of the string. + PyObject* subargs = PyTuple_New(0); + PyObject* serialized = PyUpb_Message_SerializeToString(self, subargs, NULL); + Py_DECREF(subargs); + if (!serialized) return NULL; + size_t size = PyBytes_Size(serialized); + Py_DECREF(serialized); + return PyLong_FromSize_t(size); +} + +static PyObject* PyUpb_Message_Clear(PyUpb_Message* self) { + PyUpb_Message_EnsureReified(self); + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + PyUpb_WeakMap* subobj_map = self->unset_subobj_map; + + if (subobj_map) { + upb_Message* msg = PyUpb_Message_GetMsg(self); + (void)msg; // Suppress unused warning when asserts are disabled. + intptr_t iter = PYUPB_WEAKMAP_BEGIN; + const void* key; + PyObject* obj; + + while (PyUpb_WeakMap_Next(subobj_map, &key, &obj, &iter)) { + const upb_FieldDef* f = key; + PyUpb_WeakMap_DeleteIter(subobj_map, &iter); + if (upb_FieldDef_IsMap(f)) { + assert(upb_Message_GetFieldByDef(msg, f).map_val == NULL); + PyUpb_MapContainer_Reify(obj, NULL); + } else if (upb_FieldDef_IsRepeated(f)) { + assert(upb_Message_GetFieldByDef(msg, f).array_val == NULL); + PyUpb_RepeatedContainer_Reify(obj, NULL); + } else { + assert(!upb_Message_HasFieldByDef(msg, f)); + PyUpb_Message* sub = (void*)obj; + assert(self == sub->ptr.parent); + PyUpb_Message_Reify(sub, f, NULL); + } + } + } + + upb_Message_ClearByDef(self->ptr.msg, msgdef); + Py_RETURN_NONE; +} + +void PyUpb_Message_DoClearField(PyObject* _self, const upb_FieldDef* f) { + PyUpb_Message* self = (void*)_self; + PyUpb_Message_EnsureReified((PyUpb_Message*)self); + + // We must ensure that any stub object is reified so its parent no longer + // points to us. + PyObject* sub = self->unset_subobj_map + ? PyUpb_WeakMap_Get(self->unset_subobj_map, f) + : NULL; + + if (upb_FieldDef_IsMap(f)) { + // For maps we additionally have to invalidate any iterators. So we need + // to get an object even if it's reified. + if (!sub) { + sub = PyUpb_Message_GetFieldValue(_self, f); + } + PyUpb_MapContainer_EnsureReified(sub); + PyUpb_MapContainer_Invalidate(sub); + } else if (upb_FieldDef_IsRepeated(f)) { + if (sub) { + PyUpb_RepeatedContainer_EnsureReified(sub); + } + } else if (upb_FieldDef_IsSubMessage(f)) { + if (sub) { + PyUpb_Message_EnsureReified((PyUpb_Message*)sub); + } + } + + Py_XDECREF(sub); + upb_Message_ClearFieldByDef(self->ptr.msg, f); +} + +static PyObject* PyUpb_Message_ClearExtension(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + PyUpb_Message_EnsureReified(self); + const upb_FieldDef* f = PyUpb_Message_GetExtensionDef(_self, arg); + if (!f) return NULL; + PyUpb_Message_DoClearField(_self, f); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_ClearField(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + + // We always need EnsureReified() here (even for an unset message) to + // preserve behavior like: + // msg = FooMessage() + // msg.foo.Clear() + // assert msg.HasField("foo") + PyUpb_Message_EnsureReified(self); + + const upb_FieldDef* f; + const upb_OneofDef* o; + if (!PyUpb_Message_LookupName(self, arg, &f, &o, PyExc_ValueError)) { + return NULL; + } + + if (o) f = upb_Message_WhichOneof(self->ptr.msg, o); + if (f) PyUpb_Message_DoClearField(_self, f); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_DiscardUnknownFields(PyUpb_Message* self, + PyObject* arg) { + PyUpb_Message_EnsureReified(self); + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + upb_Message_DiscardUnknown(self->ptr.msg, msgdef, 64); + Py_RETURN_NONE; +} + +static PyObject* PyUpb_Message_FindInitializationErrors(PyObject* _self, + PyObject* arg) { + PyUpb_Message* self = (void*)_self; + upb_Message* msg = PyUpb_Message_GetIfReified(_self); + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + const upb_DefPool* ext_pool = upb_FileDef_Pool(upb_MessageDef_File(msgdef)); + upb_FieldPathEntry* fields_base; + PyObject* ret = PyList_New(0); + if (upb_util_HasUnsetRequired(msg, msgdef, ext_pool, &fields_base)) { + upb_FieldPathEntry* fields = fields_base; + char* buf = NULL; + size_t size = 0; + assert(fields->field); + while (fields->field) { + upb_FieldPathEntry* field = fields; + size_t need = upb_FieldPath_ToText(&fields, buf, size); + if (need >= size) { + fields = field; + size = size ? size * 2 : 16; + while (size <= need) size *= 2; + buf = realloc(buf, size); + need = upb_FieldPath_ToText(&fields, buf, size); + assert(size > need); + } + PyObject* str = PyUnicode_FromString(buf); + PyList_Append(ret, str); + Py_DECREF(str); + } + free(buf); + free(fields_base); + } + return ret; +} + +static PyObject* PyUpb_Message_FromString(PyObject* cls, PyObject* serialized) { + PyObject* ret = NULL; + PyObject* length = NULL; + + ret = PyObject_CallObject(cls, NULL); + if (ret == NULL) goto err; + length = PyUpb_Message_MergeFromString(ret, serialized); + if (length == NULL) goto err; + +done: + Py_XDECREF(length); + return ret; + +err: + Py_XDECREF(ret); + ret = NULL; + goto done; +} + +const upb_FieldDef* PyUpb_Message_GetExtensionDef(PyObject* _self, + PyObject* key) { + const upb_FieldDef* f = PyUpb_FieldDescriptor_GetDef(key); + if (!f) { + PyErr_Clear(); + PyErr_Format(PyExc_KeyError, "Object %R is not a field descriptor\n", key); + return NULL; + } + if (!upb_FieldDef_IsExtension(f)) { + PyErr_Format(PyExc_KeyError, "Field %s is not an extension\n", + upb_FieldDef_FullName(f)); + return NULL; + } + const upb_MessageDef* msgdef = PyUpb_Message_GetMsgdef(_self); + if (upb_FieldDef_ContainingType(f) != msgdef) { + PyErr_Format(PyExc_KeyError, "Extension doesn't match (%s vs %s)", + upb_MessageDef_FullName(msgdef), upb_FieldDef_FullName(f)); + return NULL; + } + return f; +} + +static PyObject* PyUpb_Message_HasExtension(PyObject* _self, + PyObject* ext_desc) { + upb_Message* msg = PyUpb_Message_GetIfReified(_self); + const upb_FieldDef* f = PyUpb_Message_GetExtensionDef(_self, ext_desc); + if (!f) return NULL; + if (upb_FieldDef_IsRepeated(f)) { + PyErr_SetString(PyExc_KeyError, + "Field is repeated. A singular method is required."); + return NULL; + } + if (!msg) Py_RETURN_FALSE; + return PyBool_FromLong(upb_Message_HasFieldByDef(msg, f)); +} + +void PyUpb_Message_ReportInitializationErrors(const upb_MessageDef* msgdef, + PyObject* errors, PyObject* exc) { + PyObject* comma = PyUnicode_FromString(","); + PyObject* missing_fields = NULL; + if (!comma) goto done; + missing_fields = PyUnicode_Join(comma, errors); + if (!missing_fields) goto done; + PyErr_Format(exc, "Message %s is missing required fields: %U", + upb_MessageDef_FullName(msgdef), missing_fields); +done: + Py_XDECREF(comma); + Py_XDECREF(missing_fields); + Py_DECREF(errors); +} + +PyObject* PyUpb_Message_SerializeInternal(PyObject* _self, PyObject* args, + PyObject* kwargs, + bool check_required) { + PyUpb_Message* self = (void*)_self; + if (!PyUpb_Message_Verify((PyObject*)self)) return NULL; + static const char* kwlist[] = {"deterministic", NULL}; + int deterministic = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|p", (char**)(kwlist), + &deterministic)) { + return NULL; + } + + const upb_MessageDef* msgdef = _PyUpb_Message_GetMsgdef(self); + if (PyUpb_Message_IsStub(self)) { + // Nothing to serialize, but we do have to check whether the message is + // initialized. + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyObject* errors = PyUpb_Message_FindInitializationErrors(_self, NULL); + if (!errors) return NULL; + if (PyList_Size(errors) == 0) { + Py_DECREF(errors); + return PyBytes_FromStringAndSize(NULL, 0); + } + PyUpb_Message_ReportInitializationErrors(msgdef, errors, + state->encode_error_class); + return NULL; + } + + upb_Arena* arena = upb_Arena_New(); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(msgdef); + size_t size = 0; + // Python does not currently have any effective limit on serialization depth. + int options = upb_EncodeOptions_MaxDepth(UINT16_MAX); + if (check_required) options |= kUpb_EncodeOption_CheckRequired; + if (deterministic) options |= kUpb_EncodeOption_Deterministic; + char* pb; + upb_EncodeStatus status = + upb_Encode(self->ptr.msg, layout, options, arena, &pb, &size); + PyObject* ret = NULL; + + if (status != kUpb_EncodeStatus_Ok) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyObject* errors = PyUpb_Message_FindInitializationErrors(_self, NULL); + if (PyList_Size(errors) != 0) { + PyUpb_Message_ReportInitializationErrors(msgdef, errors, + state->encode_error_class); + } else { + PyErr_Format(state->encode_error_class, "Failed to serialize proto"); + } + goto done; + } + + ret = PyBytes_FromStringAndSize(pb, size); + +done: + upb_Arena_Free(arena); + return ret; +} + +PyObject* PyUpb_Message_SerializeToString(PyObject* _self, PyObject* args, + PyObject* kwargs) { + return PyUpb_Message_SerializeInternal(_self, args, kwargs, true); +} + +PyObject* PyUpb_Message_SerializePartialToString(PyObject* _self, + PyObject* args, + PyObject* kwargs) { + return PyUpb_Message_SerializeInternal(_self, args, kwargs, false); +} + +static PyObject* PyUpb_Message_WhichOneof(PyObject* _self, PyObject* name) { + PyUpb_Message* self = (void*)_self; + const upb_OneofDef* o; + if (!PyUpb_Message_LookupName(self, name, NULL, &o, PyExc_ValueError)) { + return NULL; + } + upb_Message* msg = PyUpb_Message_GetIfReified(_self); + if (!msg) Py_RETURN_NONE; + const upb_FieldDef* f = upb_Message_WhichOneof(msg, o); + if (!f) Py_RETURN_NONE; + return PyUnicode_FromString(upb_FieldDef_Name(f)); +} + +PyObject* DeepCopy(PyObject* _self, PyObject* arg) { + PyUpb_Message* self = (void*)_self; + const upb_MessageDef* def = PyUpb_Message_GetMsgdef(_self); + + PyObject* arena = PyUpb_Arena_New(); + upb_Message* clone = upb_Message_DeepClone( + self->ptr.msg, upb_MessageDef_MiniTable(def), PyUpb_Arena_Get(arena)); + PyObject* ret = PyUpb_Message_Get(clone, def, arena); + Py_DECREF(arena); + + return ret; +} + +void PyUpb_Message_ClearExtensionDict(PyObject* _self) { + PyUpb_Message* self = (void*)_self; + assert(self->ext_dict); + self->ext_dict = NULL; +} + +static PyObject* PyUpb_Message_GetExtensionDict(PyObject* _self, + void* closure) { + PyUpb_Message* self = (void*)_self; + if (self->ext_dict) { + Py_INCREF(self->ext_dict); + return self->ext_dict; + } + + const upb_MessageDef* m = _PyUpb_Message_GetMsgdef(self); + if (upb_MessageDef_ExtensionRangeCount(m) == 0) { + PyErr_SetNone(PyExc_AttributeError); + return NULL; + } + + self->ext_dict = PyUpb_ExtensionDict_New(_self); + return self->ext_dict; +} + +static PyGetSetDef PyUpb_Message_Getters[] = { + {"Extensions", PyUpb_Message_GetExtensionDict, NULL, "Extension dict"}, + {NULL}}; + +static PyMethodDef PyUpb_Message_Methods[] = { + {"__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class."}, + // TODO + //{ "__unicode__", (PyCFunction)ToUnicode, METH_NOARGS, + // "Outputs a unicode representation of the message." }, + {"ByteSize", (PyCFunction)PyUpb_Message_ByteSize, METH_NOARGS, + "Returns the size of the message in bytes."}, + {"Clear", (PyCFunction)PyUpb_Message_Clear, METH_NOARGS, + "Clears the message."}, + {"ClearExtension", PyUpb_Message_ClearExtension, METH_O, + "Clears a message field."}, + {"ClearField", PyUpb_Message_ClearField, METH_O, "Clears a message field."}, + {"CopyFrom", PyUpb_Message_CopyFrom, METH_O, + "Copies a protocol message into the current message."}, + {"DiscardUnknownFields", (PyCFunction)PyUpb_Message_DiscardUnknownFields, + METH_NOARGS, "Discards the unknown fields."}, + {"FindInitializationErrors", PyUpb_Message_FindInitializationErrors, + METH_NOARGS, "Finds unset required fields."}, + {"FromString", PyUpb_Message_FromString, METH_O | METH_CLASS, + "Creates new method instance from given serialized data."}, + {"HasExtension", PyUpb_Message_HasExtension, METH_O, + "Checks if a message field is set."}, + {"HasField", PyUpb_Message_HasField, METH_O, + "Checks if a message field is set."}, + {"IsInitialized", PyUpb_Message_IsInitialized, METH_VARARGS, + "Checks if all required fields of a protocol message are set."}, + {"ListFields", PyUpb_Message_ListFields, METH_NOARGS, + "Lists all set fields of a message."}, + {"MergeFrom", PyUpb_Message_MergeFrom, METH_O, + "Merges a protocol message into the current message."}, + {"MergeFromString", PyUpb_Message_MergeFromString, METH_O, + "Merges a serialized message into the current message."}, + {"ParseFromString", PyUpb_Message_ParseFromString, METH_O, + "Parses a serialized message into the current message."}, + {"SerializePartialToString", + (PyCFunction)PyUpb_Message_SerializePartialToString, + METH_VARARGS | METH_KEYWORDS, + "Serializes the message to a string, even if it isn't initialized."}, + {"SerializeToString", (PyCFunction)PyUpb_Message_SerializeToString, + METH_VARARGS | METH_KEYWORDS, + "Serializes the message to a string, only for initialized messages."}, + {"SetInParent", (PyCFunction)PyUpb_Message_SetInParent, METH_NOARGS, + "Sets the has bit of the given field in its parent message."}, + {"UnknownFields", (PyCFunction)PyUpb_Message_UnknownFields, METH_NOARGS, + "Parse unknown field set"}, + {"WhichOneof", PyUpb_Message_WhichOneof, METH_O, + "Returns the name of the field set inside a oneof, " + "or None if no field is set."}, + {"_ListFieldsItemKey", PyUpb_Message_ListFieldsItemKey, + METH_O | METH_STATIC, + "Compares ListFields() list entries by field number"}, + {"_CheckCalledFromGeneratedFile", + PyUpb_Message_CheckCalledFromGeneratedFile, METH_NOARGS | METH_STATIC, + "Raises TypeError if the caller is not in a _pb2.py file."}, + {NULL, NULL}}; + +static PyType_Slot PyUpb_Message_Slots[] = { + {Py_tp_dealloc, PyUpb_Message_Dealloc}, + {Py_tp_doc, "A ProtocolMessage"}, + {Py_tp_getattro, PyUpb_Message_GetAttr}, + {Py_tp_getset, PyUpb_Message_Getters}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {Py_tp_methods, PyUpb_Message_Methods}, + {Py_tp_new, PyUpb_Message_New}, + {Py_tp_str, PyUpb_Message_ToString}, + {Py_tp_repr, PyUpb_Message_ToString}, + {Py_tp_richcompare, PyUpb_Message_RichCompare}, + {Py_tp_setattro, PyUpb_Message_SetAttr}, + {Py_tp_init, PyUpb_Message_Init}, + {0, NULL}}; + +PyType_Spec PyUpb_Message_Spec = { + PYUPB_MODULE_NAME ".Message", // tp_name + sizeof(PyUpb_Message), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, // tp_flags + PyUpb_Message_Slots, +}; + +// ----------------------------------------------------------------------------- +// MessageMeta +// ----------------------------------------------------------------------------- + +// MessageMeta is the metaclass for message objects. The generated code uses it +// to construct message classes, ie. +// +// FooMessage = _message.MessageMeta('FooMessage', (_message.Message), {...}) +// +// (This is not quite true: at the moment the Python library subclasses +// MessageMeta, and uses that subclass as the metaclass. There is a TODO below +// to simplify this, so that the illustration above is indeed accurate). + +typedef struct { + const upb_MiniTable* layout; + PyObject* py_message_descriptor; +} PyUpb_MessageMeta; + +// The PyUpb_MessageMeta struct is trailing data tacked onto the end of +// MessageMeta instances. This means that we get our instances of this struct +// by adding the appropriate number of bytes. +static PyUpb_MessageMeta* PyUpb_GetMessageMeta(PyObject* cls) { +#ifndef NDEBUG + PyUpb_ModuleState* state = PyUpb_ModuleState_MaybeGet(); + assert(!state || cls->ob_type == state->message_meta_type); +#endif + return (PyUpb_MessageMeta*)((char*)cls + cpython_bits.type_basicsize); +} + +static const upb_MessageDef* PyUpb_MessageMeta_GetMsgdef(PyObject* cls) { + PyUpb_MessageMeta* self = PyUpb_GetMessageMeta(cls); + return PyUpb_Descriptor_GetDef(self->py_message_descriptor); +} + +PyObject* PyUpb_MessageMeta_DoCreateClass(PyObject* py_descriptor, + const char* name, PyObject* dict) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyTypeObject* descriptor_type = state->descriptor_types[kPyUpb_Descriptor]; + if (!PyObject_TypeCheck(py_descriptor, descriptor_type)) { + return PyErr_Format(PyExc_TypeError, "Expected a message Descriptor"); + } + + const upb_MessageDef* msgdef = PyUpb_Descriptor_GetDef(py_descriptor); + assert(msgdef); + assert(!PyUpb_ObjCache_Get(upb_MessageDef_MiniTable(msgdef))); + + PyObject* slots = PyTuple_New(0); + if (!slots) return NULL; + int status = PyDict_SetItemString(dict, "__slots__", slots); + Py_DECREF(slots); + if (status < 0) return NULL; + + // Bases are either: + // (Message, Message) # for regular messages + // (Message, Message, WktBase) # For well-known types + PyObject* wkt_bases = PyUpb_GetWktBases(state); + PyObject* wkt_base = + PyDict_GetItemString(wkt_bases, upb_MessageDef_FullName(msgdef)); + PyObject* args; + if (wkt_base == NULL) { + args = Py_BuildValue("s(OO)O", name, state->cmessage_type, + state->message_class, dict); + } else { + args = Py_BuildValue("s(OOO)O", name, state->cmessage_type, + state->message_class, wkt_base, dict); + } + + PyObject* ret = cpython_bits.type_new(state->message_meta_type, args, NULL); + Py_DECREF(args); + if (!ret) return NULL; + + PyUpb_MessageMeta* meta = PyUpb_GetMessageMeta(ret); + meta->py_message_descriptor = py_descriptor; + meta->layout = upb_MessageDef_MiniTable(msgdef); + Py_INCREF(meta->py_message_descriptor); + PyUpb_Descriptor_SetClass(py_descriptor, ret); + + PyUpb_ObjCache_Add(meta->layout, ret); + + return ret; +} + +static PyObject* PyUpb_MessageMeta_New(PyTypeObject* type, PyObject* args, + PyObject* kwargs) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + static const char* kwlist[] = {"name", "bases", "dict", 0}; + PyObject *bases, *dict; + const char* name; + + // Check arguments: (name, bases, dict) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "sO!O!:type", (char**)kwlist, + &name, &PyTuple_Type, &bases, &PyDict_Type, + &dict)) { + return NULL; + } + + // Check bases: only (), or (message.Message,) are allowed + Py_ssize_t size = PyTuple_Size(bases); + if (!(size == 0 || + (size == 1 && PyTuple_GetItem(bases, 0) == state->message_class))) { + PyErr_Format(PyExc_TypeError, + "A Message class can only inherit from Message, not %S", + bases); + return NULL; + } + + // Check dict['DESCRIPTOR'] + PyObject* py_descriptor = PyDict_GetItemString(dict, "DESCRIPTOR"); + if (py_descriptor == NULL) { + PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR"); + return NULL; + } + + const upb_MessageDef* m = PyUpb_Descriptor_GetDef(py_descriptor); + PyObject* ret = PyUpb_ObjCache_Get(upb_MessageDef_MiniTable(m)); + if (ret) return ret; + return PyUpb_MessageMeta_DoCreateClass(py_descriptor, name, dict); +} + +static void PyUpb_MessageMeta_Dealloc(PyObject* self) { + PyUpb_MessageMeta* meta = PyUpb_GetMessageMeta(self); + PyUpb_ObjCache_Delete(meta->layout); + Py_DECREF(meta->py_message_descriptor); + PyTypeObject* tp = Py_TYPE(self); + cpython_bits.type_dealloc(self); + Py_DECREF(tp); +} + +void PyUpb_MessageMeta_AddFieldNumber(PyObject* self, const upb_FieldDef* f) { + PyObject* name = + PyUnicode_FromFormat("%s_FIELD_NUMBER", upb_FieldDef_Name(f)); + PyObject* upper = PyObject_CallMethod(name, "upper", ""); + PyObject_SetAttr(self, upper, PyLong_FromLong(upb_FieldDef_Number(f))); + Py_DECREF(name); + Py_DECREF(upper); +} + +static PyObject* PyUpb_MessageMeta_GetDynamicAttr(PyObject* self, + PyObject* name) { + const char* name_buf = PyUpb_GetStrData(name); + if (!name_buf) return NULL; + const upb_MessageDef* msgdef = PyUpb_MessageMeta_GetMsgdef(self); + const upb_FileDef* filedef = upb_MessageDef_File(msgdef); + const upb_DefPool* symtab = upb_FileDef_Pool(filedef); + + PyObject* py_key = + PyBytes_FromFormat("%s.%s", upb_MessageDef_FullName(msgdef), name_buf); + const char* key = PyUpb_GetStrData(py_key); + PyObject* ret = NULL; + const upb_MessageDef* nested = upb_DefPool_FindMessageByName(symtab, key); + const upb_EnumDef* enumdef; + const upb_EnumValueDef* enumval; + const upb_FieldDef* ext; + + if (nested) { + ret = PyUpb_Descriptor_GetClass(nested); + } else if ((enumdef = upb_DefPool_FindEnumByName(symtab, key))) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyObject* klass = state->enum_type_wrapper_class; + ret = PyUpb_EnumDescriptor_Get(enumdef); + ret = PyObject_CallFunctionObjArgs(klass, ret, NULL); + } else if ((enumval = upb_DefPool_FindEnumByNameval(symtab, key))) { + ret = PyLong_FromLong(upb_EnumValueDef_Number(enumval)); + } else if ((ext = upb_DefPool_FindExtensionByName(symtab, key))) { + ret = PyUpb_FieldDescriptor_Get(ext); + } + + Py_DECREF(py_key); + + const char* suffix = "_FIELD_NUMBER"; + size_t n = strlen(name_buf); + size_t suffix_n = strlen(suffix); + if (n > suffix_n && memcmp(suffix, name_buf + n - suffix_n, suffix_n) == 0) { + // We can't look up field names dynamically, because the <NAME>_FIELD_NUMBER + // naming scheme upper-cases the field name and is therefore non-reversible. + // So we just add all field numbers. + int n = upb_MessageDef_FieldCount(msgdef); + for (int i = 0; i < n; i++) { + PyUpb_MessageMeta_AddFieldNumber(self, upb_MessageDef_Field(msgdef, i)); + } + n = upb_MessageDef_NestedExtensionCount(msgdef); + for (int i = 0; i < n; i++) { + PyUpb_MessageMeta_AddFieldNumber( + self, upb_MessageDef_NestedExtension(msgdef, i)); + } + ret = PyObject_GenericGetAttr(self, name); + } + + return ret; +} + +static PyObject* PyUpb_MessageMeta_GetAttr(PyObject* self, PyObject* name) { + // We want to first delegate to the type's tp_dict to retrieve any attributes + // that were previously calculated and cached in the type's dict. + PyObject* ret = cpython_bits.type_getattro(self, name); + if (ret) return ret; + + // We did not find a cached attribute. Try to calculate the attribute + // dynamically, using the descriptor as an argument. + PyErr_Clear(); + ret = PyUpb_MessageMeta_GetDynamicAttr(self, name); + + if (ret) { + PyObject_SetAttr(self, name, ret); + PyErr_Clear(); + return ret; + } + + PyErr_SetObject(PyExc_AttributeError, name); + return NULL; +} + +static int PyUpb_MessageMeta_Traverse(PyObject* self, visitproc visit, + void* arg) { + PyUpb_MessageMeta* meta = PyUpb_GetMessageMeta(self); + Py_VISIT(meta->py_message_descriptor); + return cpython_bits.type_traverse(self, visit, arg); +} + +static int PyUpb_MessageMeta_Clear(PyObject* self, visitproc visit, void* arg) { + return cpython_bits.type_clear(self); +} + +static PyType_Slot PyUpb_MessageMeta_Slots[] = { + {Py_tp_new, PyUpb_MessageMeta_New}, + {Py_tp_dealloc, PyUpb_MessageMeta_Dealloc}, + {Py_tp_getattro, PyUpb_MessageMeta_GetAttr}, + {Py_tp_traverse, PyUpb_MessageMeta_Traverse}, + {Py_tp_clear, PyUpb_MessageMeta_Clear}, + {0, NULL}}; + +static PyType_Spec PyUpb_MessageMeta_Spec = { + PYUPB_MODULE_NAME ".MessageMeta", // tp_name + 0, // To be filled in by size of base // tp_basicsize + 0, // tp_itemsize + // TODO: remove BASETYPE, Python should just use MessageMeta + // directly instead of subclassing it. + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, // tp_flags + PyUpb_MessageMeta_Slots, +}; + +static PyObject* PyUpb_MessageMeta_CreateType(void) { + PyObject* bases = Py_BuildValue("(O)", &PyType_Type); + if (!bases) return NULL; + PyUpb_MessageMeta_Spec.basicsize = + cpython_bits.type_basicsize + sizeof(PyUpb_MessageMeta); + PyObject* type = PyType_FromSpecWithBases(&PyUpb_MessageMeta_Spec, bases); + Py_DECREF(bases); + return type; +} + +bool PyUpb_InitMessage(PyObject* m) { + if (!PyUpb_CPythonBits_Init(&cpython_bits)) return false; + PyObject* message_meta_type = PyUpb_MessageMeta_CreateType(); + + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + state->cmessage_type = PyUpb_AddClass(m, &PyUpb_Message_Spec); + state->message_meta_type = (PyTypeObject*)message_meta_type; + + if (!state->cmessage_type || !state->message_meta_type) return false; + if (PyModule_AddObject(m, "MessageMeta", message_meta_type)) return false; + state->listfields_item_key = PyObject_GetAttrString( + (PyObject*)state->cmessage_type, "_ListFieldsItemKey"); + + PyObject* mod = + PyImport_ImportModule(PYUPB_PROTOBUF_PUBLIC_PACKAGE ".message"); + if (mod == NULL) return false; + + state->encode_error_class = PyObject_GetAttrString(mod, "EncodeError"); + state->decode_error_class = PyObject_GetAttrString(mod, "DecodeError"); + state->message_class = PyObject_GetAttrString(mod, "Message"); + Py_DECREF(mod); + + PyObject* enum_type_wrapper = PyImport_ImportModule( + PYUPB_PROTOBUF_INTERNAL_PACKAGE ".enum_type_wrapper"); + if (enum_type_wrapper == NULL) return false; + + state->enum_type_wrapper_class = + PyObject_GetAttrString(enum_type_wrapper, "EnumTypeWrapper"); + Py_DECREF(enum_type_wrapper); + + if (!state->encode_error_class || !state->decode_error_class || + !state->message_class || !state->listfields_item_key || + !state->enum_type_wrapper_class) { + return false; + } + + return true; +}
diff --git a/python/message.h b/python/message.h new file mode 100644 index 0000000..d497f61 --- /dev/null +++ b/python/message.h
@@ -0,0 +1,110 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYPB_MESSAGE_H__ +#define PYPB_MESSAGE_H__ + +#include <stdbool.h> + +#include "python/protobuf.h" +#include "upb/reflection/message.h" + +// Removes the wrapper object for this field from the unset subobject cache. +void PyUpb_Message_CacheDelete(PyObject* _self, const upb_FieldDef* f); + +// Sets the field value for `f` to `subobj`, evicting the wrapper object from +// the "unset subobject" cache now that real data exists for it. The caller +// must also update the wrapper associated with `f` to point to `subobj` also. +void PyUpb_Message_SetConcreteSubobj(PyObject* _self, const upb_FieldDef* f, + upb_MessageValue subobj); + +// Gets a Python wrapper object for message `u_msg` of type `m`, returning a +// cached wrapper if one was previously created. If a new object is created, +// it will reference `arena`, which must own `u_msg`. +PyObject* PyUpb_Message_Get(upb_Message* u_msg, const upb_MessageDef* m, + PyObject* arena); + +// Verifies that a Python object is a message. Sets a TypeError exception and +// returns false on failure. +bool PyUpb_Message_Verify(PyObject* self); + +// Gets the upb_Message* for this message object if the message is reified. +// Otherwise returns NULL. +upb_Message* PyUpb_Message_GetIfReified(PyObject* _self); + +// Returns the `upb_MessageDef` for a given Message. +const upb_MessageDef* PyUpb_Message_GetMsgdef(PyObject* self); + +// Functions that match the corresponding methods on the message object. +PyObject* PyUpb_Message_MergeFrom(PyObject* self, PyObject* arg); +PyObject* PyUpb_Message_MergeFromString(PyObject* self, PyObject* arg); +PyObject* PyUpb_Message_SerializeToString(PyObject* self, PyObject* args, + PyObject* kwargs); +PyObject* PyUpb_Message_SerializePartialToString(PyObject* self, PyObject* args, + PyObject* kwargs); + +// Sets fields of the message according to the attribuges in `kwargs`. +int PyUpb_Message_InitAttributes(PyObject* _self, PyObject* args, + PyObject* kwargs); + +// Checks that `key` is a field descriptor for an extension type, and that the +// extendee is this message. Otherwise returns NULL and sets a KeyError. +const upb_FieldDef* PyUpb_Message_GetExtensionDef(PyObject* _self, + PyObject* key); + +// Clears the given field in this message. +void PyUpb_Message_DoClearField(PyObject* _self, const upb_FieldDef* f); + +// Clears the ExtensionDict from the message. The message must have an +// ExtensionDict set. +void PyUpb_Message_ClearExtensionDict(PyObject* _self); + +// Implements the equivalent of getattr(msg, field), once `field` has +// already been resolved to a `upb_FieldDef*`. +PyObject* PyUpb_Message_GetFieldValue(PyObject* _self, + const upb_FieldDef* field); + +// Implements the equivalent of setattr(msg, field, value), once `field` has +// already been resolved to a `upb_FieldDef*`. +int PyUpb_Message_SetFieldValue(PyObject* _self, const upb_FieldDef* field, + PyObject* value, PyObject* exc); + +// Creates message meta class. +PyObject* PyUpb_MessageMeta_DoCreateClass(PyObject* py_descriptor, + const char* name, PyObject* dict); + +// Returns the version associated with this message. The version will be +// incremented when the message changes. +int PyUpb_Message_GetVersion(PyObject* _self); + +// Module-level init. +bool PyUpb_InitMessage(PyObject* m); + +#endif // PYPB_MESSAGE_H__
diff --git a/python/minimal_test.py b/python/minimal_test.py new file mode 100644 index 0000000..e1690d1 --- /dev/null +++ b/python/minimal_test.py
@@ -0,0 +1,187 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A bare-bones unit test that doesn't load any generated code.""" + + +import unittest +from google.protobuf.pyext import _message +from google3.net.proto2.python.internal import api_implementation +from google.protobuf import unittest_pb2 +from google.protobuf import map_unittest_pb2 +from google.protobuf import descriptor_pool +from google.protobuf import text_format +from google.protobuf import message_factory +from google.protobuf import message +from google3.net.proto2.python.internal import factory_test1_pb2 +from google3.net.proto2.python.internal import factory_test2_pb2 +from google3.net.proto2.python.internal import more_extensions_pb2 +from google.protobuf import descriptor_pb2 + +class TestMessageExtension(unittest.TestCase): + + def test_descriptor_pool(self): + serialized_desc = b'\n\ntest.proto\"\x0e\n\x02M1*\x08\x08\x01\x10\x80\x80\x80\x80\x02:\x15\n\x08test_ext\x12\x03.M1\x18\x01 \x01(\x05' + pool = _message.DescriptorPool() + file_desc = pool.AddSerializedFile(serialized_desc) + self.assertEqual("test.proto", file_desc.name) + ext_desc = pool.FindExtensionByName("test_ext") + self.assertEqual(1, ext_desc.number) + + # Test object cache: repeatedly retrieving the same descriptor + # should result in the same object + self.assertIs(ext_desc, pool.FindExtensionByName("test_ext")) + + + def test_lib_is_upb(self): + # Ensure we are not pulling in a different protobuf library on the + # system. + print(_message._IS_UPB) + self.assertTrue(_message._IS_UPB) + self.assertEqual(api_implementation.Type(), "cpp") + + def test_repeated_field_slice_delete(self): + def test_slice(start, end, step): + vals = list(range(20)) + message = unittest_pb2.TestAllTypes(repeated_int32=vals) + del vals[start:end:step] + del message.repeated_int32[start:end:step] + self.assertEqual(vals, list(message.repeated_int32)) + test_slice(3, 11, 1) + test_slice(3, 11, 2) + test_slice(3, 11, 3) + test_slice(11, 3, -1) + test_slice(11, 3, -2) + test_slice(11, 3, -3) + test_slice(10, 25, 4) + + def testExtensionsErrors(self): + msg = unittest_pb2.TestAllTypes() + self.assertRaises(AttributeError, getattr, msg, 'Extensions') + + def testClearStubMapField(self): + msg = map_unittest_pb2.TestMapSubmessage() + int32_map = msg.test_map.map_int32_int32 + msg.test_map.ClearField("map_int32_int32") + int32_map[123] = 456 + self.assertEqual(0, msg.test_map.ByteSize()) + + def testClearReifiedMapField(self): + msg = map_unittest_pb2.TestMap() + int32_map = msg.map_int32_int32 + int32_map[123] = 456 + msg.ClearField("map_int32_int32") + int32_map[111] = 222 + self.assertEqual(0, msg.ByteSize()) + + def testClearStubRepeatedField(self): + msg = unittest_pb2.NestedTestAllTypes() + int32_array = msg.payload.repeated_int32 + msg.payload.ClearField("repeated_int32") + int32_array.append(123) + self.assertEqual(0, msg.payload.ByteSize()) + + def testClearReifiedRepeatdField(self): + msg = unittest_pb2.TestAllTypes() + int32_array = msg.repeated_int32 + int32_array.append(123) + self.assertNotEqual(0, msg.ByteSize()) + msg.ClearField("repeated_int32") + int32_array.append(123) + self.assertEqual(0, msg.ByteSize()) + + def testFloatPrinting(self): + message = unittest_pb2.TestAllTypes() + message.optional_float = -0.0 + self.assertEqual(str(message), 'optional_float: -0\n') + +class OversizeProtosTest(unittest.TestCase): + def setUp(self): + msg = unittest_pb2.NestedTestAllTypes() + m = msg + for i in range(101): + m = m.child + m.Clear() + self.p_serialized = msg.SerializeToString() + + def testAssertOversizeProto(self): + from google.protobuf.pyext._message import SetAllowOversizeProtos + SetAllowOversizeProtos(False) + q = unittest_pb2.NestedTestAllTypes() + with self.assertRaises(message.DecodeError): + q.ParseFromString(self.p_serialized) + print(q) + + def testSucceedOversizeProto(self): + from google.protobuf.pyext._message import SetAllowOversizeProtos + SetAllowOversizeProtos(True) + q = unittest_pb2.NestedTestAllTypes() + q.ParseFromString(self.p_serialized) + + def testExtensionIter(self): + extendee_proto = more_extensions_pb2.ExtendedMessage() + + extension_int32 = more_extensions_pb2.optional_int_extension + extendee_proto.Extensions[extension_int32] = 23 + + extension_repeated = more_extensions_pb2.repeated_int_extension + extendee_proto.Extensions[extension_repeated].append(11) + + extension_msg = more_extensions_pb2.optional_message_extension + extendee_proto.Extensions[extension_msg].foreign_message_int = 56 + + # Set some normal fields. + extendee_proto.optional_int32 = 1 + extendee_proto.repeated_string.append('hi') + + expected = { + extension_int32: True, + extension_msg: True, + extension_repeated: True + } + count = 0 + for item in extendee_proto.Extensions: + del expected[item] + self.assertIn(item, extendee_proto.Extensions) + count += 1 + self.assertEqual(count, 3) + self.assertEqual(len(expected), 0) + + def testIsInitializedStub(self): + proto = unittest_pb2.TestRequiredForeign() + self.assertTrue(proto.IsInitialized()) + self.assertFalse(proto.optional_message.IsInitialized()) + errors = [] + self.assertFalse(proto.optional_message.IsInitialized(errors)) + self.assertEqual(['a', 'b', 'c'], errors) + self.assertRaises(message.EncodeError, proto.optional_message.SerializeToString) + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/BUILD b/python/pb_unit_tests/BUILD new file mode 100644 index 0000000..56ee250 --- /dev/null +++ b/python/pb_unit_tests/BUILD
@@ -0,0 +1,84 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load(":pyproto_test_wrapper.bzl", "pyproto_test_wrapper") +# begin:github_only +load("@pip_deps//:requirements.bzl", "requirement") +# end:github_only + +# begin:google_only +# package(default_applicable_licenses = ["//upb:license"]) +# end:google_only + +licenses(["notice"]) + +pyproto_test_wrapper(name = "descriptor_database_test") + +pyproto_test_wrapper(name = "descriptor_pool_test") + +pyproto_test_wrapper(name = "descriptor_test") + +# begin:github_only +pyproto_test_wrapper(name = "generator_test") +# end:github_only + +pyproto_test_wrapper(name = "json_format_test") + +pyproto_test_wrapper(name = "keywords_test") + +pyproto_test_wrapper(name = "message_factory_test") + +# begin:github_only +# This target has different dependencies and fails when using the wrapper +# TODO: Move this to using pyproto_test_wrapper +py_test( + name = "numpy_test", + srcs = ["numpy_test_wrapper.py"], + main = "numpy_test_wrapper.py", + deps = [ + requirement("numpy"), + "//python/google/protobuf/internal/numpy:numpy_test", + "//python:_message", + ], + target_compatible_with = select({ + "@system_python//:supported": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) +# end:github_only + +# begin:google_only +# pyproto_test_wrapper(name = "numpy_test") +# end:google_only + +pyproto_test_wrapper(name = "proto_builder_test") + +pyproto_test_wrapper(name = "service_reflection_test") + +pyproto_test_wrapper(name = "symbol_database_test") + +pyproto_test_wrapper(name = "text_encoding_test") + +pyproto_test_wrapper(name = "message_test") + +pyproto_test_wrapper(name = "reflection_test") + +pyproto_test_wrapper(name = "text_format_test") + +pyproto_test_wrapper(name = "unknown_fields_test") + +pyproto_test_wrapper(name = "well_known_types_test") + +pyproto_test_wrapper(name = "wire_format_test") + +filegroup( + name = "test_files", + srcs = glob(["*.py"]), + visibility = [ + "//python/dist:__pkg__", # Scheuklappen: keep + ], +)
diff --git a/python/pb_unit_tests/README.md b/python/pb_unit_tests/README.md new file mode 100644 index 0000000..669f067 --- /dev/null +++ b/python/pb_unit_tests/README.md
@@ -0,0 +1,11 @@ + +# Protobuf Unit Tests + +This directory contains wrappers around the Python unit tests defined in +the protobuf repo. Python+upb is intended to be a drop-in replacement for +protobuf Python, so we should be able to pass the same set of unit tests. + +Our wrappers contain exclusion lists for tests we know we are not currently +passing. Ideally these exclusion lists will become empty once Python+upb is +fully implemented. However there may be a few edge cases that we decide +are not worth matching with perfect parity.
diff --git a/python/pb_unit_tests/descriptor_database_test_wrapper.py b/python/pb_unit_tests/descriptor_database_test_wrapper.py new file mode 100644 index 0000000..2e6081f --- /dev/null +++ b/python/pb_unit_tests/descriptor_database_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.descriptor_database_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/descriptor_pool_test_wrapper.py b/python/pb_unit_tests/descriptor_pool_test_wrapper.py new file mode 100644 index 0000000..1c4f282 --- /dev/null +++ b/python/pb_unit_tests/descriptor_pool_test_wrapper.py
@@ -0,0 +1,45 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +from google.protobuf.internal.descriptor_pool_test import * + +SecondaryDescriptorFromDescriptorDB.testErrorCollector.__unittest_expecting_failure__ = True + +# begin:github_only +if __name__ == '__main__': + unittest.main(verbosity=2) +# end:github_only + +# begin:google_only +# from absl import app +# if __name__ == '__main__': +# app.run(lambda argv: unittest.main(verbosity=2)) +# end:google_only
diff --git a/python/pb_unit_tests/descriptor_test_wrapper.py b/python/pb_unit_tests/descriptor_test_wrapper.py new file mode 100644 index 0000000..11f47ad --- /dev/null +++ b/python/pb_unit_tests/descriptor_test_wrapper.py
@@ -0,0 +1,46 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.descriptor_test import * +import unittest + +# These fail because they attempt to add fields with conflicting JSON names. +# We don't want to support this going forward. +MakeDescriptorTest.testCamelcaseName.__unittest_expecting_failure__ = True +MakeDescriptorTest.testJsonName.__unittest_expecting_failure__ = True + +# We pass this test, but the error message is slightly different. +# Our error message is better. +NewDescriptorTest.testImmutableCppDescriptor.__unittest_expecting_failure__ = True + +DescriptorTest.testGetDebugString.__unittest_expecting_failure__ = True + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/generator_test_wrapper.py b/python/pb_unit_tests/generator_test_wrapper.py new file mode 100644 index 0000000..9ffc27f --- /dev/null +++ b/python/pb_unit_tests/generator_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.generator_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/json_format_test_wrapper.py b/python/pb_unit_tests/json_format_test_wrapper.py new file mode 100644 index 0000000..27d855c --- /dev/null +++ b/python/pb_unit_tests/json_format_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.json_format_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/keywords_test_wrapper.py b/python/pb_unit_tests/keywords_test_wrapper.py new file mode 100644 index 0000000..d940178 --- /dev/null +++ b/python/pb_unit_tests/keywords_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.keywords_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/message_factory_test_wrapper.py b/python/pb_unit_tests/message_factory_test_wrapper.py new file mode 100644 index 0000000..4e3a7ba --- /dev/null +++ b/python/pb_unit_tests/message_factory_test_wrapper.py
@@ -0,0 +1,37 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.message_factory_test import * +import unittest + +MessageFactoryTest.testDuplicateExtensionNumber.__unittest_expecting_failure__ = True + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/message_test_wrapper.py b/python/pb_unit_tests/message_test_wrapper.py new file mode 100644 index 0000000..fcac3a3 --- /dev/null +++ b/python/pb_unit_tests/message_test_wrapper.py
@@ -0,0 +1,55 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.message_test import * +import unittest + +MessageTest.testExtendFloatWithNothing_proto2.__unittest_skip__ = True +MessageTest.testExtendFloatWithNothing_proto3.__unittest_skip__ = True +MessageTest.testExtendInt32WithNothing_proto2.__unittest_skip__ = True +MessageTest.testExtendInt32WithNothing_proto3.__unittest_skip__ = True +MessageTest.testExtendStringWithNothing_proto2.__unittest_skip__ = True +MessageTest.testExtendStringWithNothing_proto3.__unittest_skip__ = True + +# Python/C++ customizes the C++ TextFormat to always print trailing ".0" for +# floats. upb doesn't do this, it matches C++ TextFormat. +MessageTest.testFloatPrinting_proto2.__unittest_expecting_failure__ = True +MessageTest.testFloatPrinting_proto3.__unittest_expecting_failure__ = True + +# For these tests we are throwing the correct error, only the text of the error +# message is a mismatch. For technical reasons around the limited API, matching +# the existing error message exactly is not feasible. +Proto3Test.testCopyFromBadType.__unittest_expecting_failure__ = True +Proto3Test.testMergeFromBadType.__unittest_expecting_failure__ = True + +Proto2Test.test_documentation.__unittest_expecting_failure__ = True + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/numpy_test_wrapper.py b/python/pb_unit_tests/numpy_test_wrapper.py new file mode 100644 index 0000000..62089e9 --- /dev/null +++ b/python/pb_unit_tests/numpy_test_wrapper.py
@@ -0,0 +1,36 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from google.protobuf.internal.numpy.numpy_test import * + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/proto_builder_test_wrapper.py b/python/pb_unit_tests/proto_builder_test_wrapper.py new file mode 100644 index 0000000..468d13e --- /dev/null +++ b/python/pb_unit_tests/proto_builder_test_wrapper.py
@@ -0,0 +1,37 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.proto_builder_test import * +import unittest + +ProtoBuilderTest.testMakeLargeProtoClass.__unittest_expecting_failure__ = True + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/pyproto_test_wrapper.bzl b/python/pb_unit_tests/pyproto_test_wrapper.bzl new file mode 100644 index 0000000..5691e88 --- /dev/null +++ b/python/pb_unit_tests/pyproto_test_wrapper.bzl
@@ -0,0 +1,46 @@ +# begin:github_only + +def pyproto_test_wrapper(name, deps = []): + src = name + "_wrapper.py" + native.py_test( + name = name, + srcs = [src], + legacy_create_init = False, + main = src, + data = ["//src/google/protobuf:testdata"], + deps = [ + "//python:_message", + "//:python_common_test_protos", + "//:python_specific_test_protos", + "//:python_test_srcs", + "//:python_srcs", + ] + deps, + target_compatible_with = select({ + "@system_python//:supported": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + ) + +# end:github_only + +# begin:google_only +# +# load("//third_party/bazel_rules/rules_python/python:py_test.bzl", "py_test") +# +# def pyproto_test_wrapper(name): +# src = name + "_wrapper.py" +# py_test( +# name = name, +# srcs = [src], +# main = src, +# deps = [ +# "//third_party/py/google/protobuf/internal:" + name + "_for_deps", +# "//net/proto2/python/public:use_upb_protos", +# ], +# target_compatible_with = select({ +# "@platforms//os:windows": ["@platforms//:incompatible"], +# "//conditions:default": [], +# }), +# ) +# +# end:google_only
diff --git a/python/pb_unit_tests/reflection_test_wrapper.py b/python/pb_unit_tests/reflection_test_wrapper.py new file mode 100644 index 0000000..7572f7c --- /dev/null +++ b/python/pb_unit_tests/reflection_test_wrapper.py
@@ -0,0 +1,53 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.reflection_test import * +import unittest + +# These tests depend on a specific iteration order for extensions, which is not +# reasonable to guarantee. +Proto2ReflectionTest.testExtensionIter.__unittest_expecting_failure__ = True + +# These tests depend on a specific serialization order for extensions, which is +# not reasonable to guarantee. +SerializationTest.testCanonicalSerializationOrder.__unittest_expecting_failure__ = True +SerializationTest.testCanonicalSerializationOrderSameAsCpp.__unittest_expecting_failure__ = True + +# This test relies on the internal implementation using Python descriptors. +# This is an implementation detail that users should not depend on. +SerializationTest.testFieldDataDescriptor.__unittest_expecting_failure__ = True + +SerializationTest.testFieldProperties.__unittest_expecting_failure__ = True + +# TODO Python Docker image on MacOS failing. +ClassAPITest.testParsingNestedClass.__unittest_skip__ = True + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/service_reflection_test_wrapper.py b/python/pb_unit_tests/service_reflection_test_wrapper.py new file mode 100644 index 0000000..bc0345c --- /dev/null +++ b/python/pb_unit_tests/service_reflection_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.service_reflection_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/symbol_database_test_wrapper.py b/python/pb_unit_tests/symbol_database_test_wrapper.py new file mode 100644 index 0000000..16ea965 --- /dev/null +++ b/python/pb_unit_tests/symbol_database_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.symbol_database_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/text_encoding_test_wrapper.py b/python/pb_unit_tests/text_encoding_test_wrapper.py new file mode 100644 index 0000000..3eb8153 --- /dev/null +++ b/python/pb_unit_tests/text_encoding_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.text_encoding_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/text_format_test_wrapper.py b/python/pb_unit_tests/text_format_test_wrapper.py new file mode 100644 index 0000000..535561d --- /dev/null +++ b/python/pb_unit_tests/text_format_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.text_format_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/unknown_fields_test_wrapper.py b/python/pb_unit_tests/unknown_fields_test_wrapper.py new file mode 100644 index 0000000..1807f6d --- /dev/null +++ b/python/pb_unit_tests/unknown_fields_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.unknown_fields_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/well_known_types_test_wrapper.py b/python/pb_unit_tests/well_known_types_test_wrapper.py new file mode 100644 index 0000000..5006332 --- /dev/null +++ b/python/pb_unit_tests/well_known_types_test_wrapper.py
@@ -0,0 +1,36 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.well_known_types_test import * +import os +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/pb_unit_tests/wire_format_test_wrapper.py b/python/pb_unit_tests/wire_format_test_wrapper.py new file mode 100644 index 0000000..3b13a2b --- /dev/null +++ b/python/pb_unit_tests/wire_format_test_wrapper.py
@@ -0,0 +1,35 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from google.protobuf.internal.wire_format_test import * +import unittest + +if __name__ == '__main__': + unittest.main(verbosity=2)
diff --git a/python/protobuf.c b/python/protobuf.c new file mode 100644 index 0000000..324b1ed --- /dev/null +++ b/python/protobuf.c
@@ -0,0 +1,431 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/protobuf.h" + +#include "python/descriptor.h" +#include "python/descriptor_containers.h" +#include "python/descriptor_pool.h" +#include "python/extension_dict.h" +#include "python/map.h" +#include "python/message.h" +#include "python/repeated.h" +#include "python/unknown_fields.h" + +static upb_Arena* PyUpb_NewArena(void); + +static void PyUpb_ModuleDealloc(void* module) { + PyUpb_ModuleState* s = PyModule_GetState(module); + PyUpb_WeakMap_Free(s->obj_cache); + if (s->c_descriptor_symtab) { + upb_DefPool_Free(s->c_descriptor_symtab); + } +} + +PyObject* PyUpb_SetAllowOversizeProtos(PyObject* m, PyObject* arg) { + if (!arg || !PyBool_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "Argument to SetAllowOversizeProtos must be boolean"); + return NULL; + } + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + state->allow_oversize_protos = PyObject_IsTrue(arg); + Py_INCREF(arg); + return arg; +} + +static PyMethodDef PyUpb_ModuleMethods[] = { + {"SetAllowOversizeProtos", PyUpb_SetAllowOversizeProtos, METH_O, + "Enable/disable oversize proto parsing."}, + {NULL, NULL}}; + +static struct PyModuleDef module_def = {PyModuleDef_HEAD_INIT, + PYUPB_MODULE_NAME, + "Protobuf Module", + sizeof(PyUpb_ModuleState), + PyUpb_ModuleMethods, // m_methods + NULL, // m_slots + NULL, // m_traverse + NULL, // m_clear + PyUpb_ModuleDealloc}; + +// ----------------------------------------------------------------------------- +// ModuleState +// ----------------------------------------------------------------------------- + +PyUpb_ModuleState* PyUpb_ModuleState_MaybeGet(void) { + PyObject* module = PyState_FindModule(&module_def); + return module ? PyModule_GetState(module) : NULL; +} + +PyUpb_ModuleState* PyUpb_ModuleState_GetFromModule(PyObject* module) { + PyUpb_ModuleState* state = PyModule_GetState(module); + assert(state); + assert(PyModule_GetDef(module) == &module_def); + return state; +} + +PyUpb_ModuleState* PyUpb_ModuleState_Get(void) { + PyObject* module = PyState_FindModule(&module_def); + assert(module); + return PyUpb_ModuleState_GetFromModule(module); +} + +PyObject* PyUpb_GetWktBases(PyUpb_ModuleState* state) { + if (!state->wkt_bases) { + PyObject* wkt_module = PyImport_ImportModule(PYUPB_PROTOBUF_INTERNAL_PACKAGE + ".well_known_types"); + + if (wkt_module == NULL) { + return false; + } + + state->wkt_bases = PyObject_GetAttrString(wkt_module, "WKTBASES"); + PyObject* m = PyState_FindModule(&module_def); + // Reparent ownership to m. + PyModule_AddObject(m, "__internal_wktbases", state->wkt_bases); + Py_DECREF(wkt_module); + } + + return state->wkt_bases; +} + +// ----------------------------------------------------------------------------- +// WeakMap +// ----------------------------------------------------------------------------- + +struct PyUpb_WeakMap { + upb_inttable table; + upb_Arena* arena; +}; + +PyUpb_WeakMap* PyUpb_WeakMap_New(void) { + upb_Arena* arena = PyUpb_NewArena(); + PyUpb_WeakMap* map = upb_Arena_Malloc(arena, sizeof(*map)); + map->arena = arena; + upb_inttable_init(&map->table, map->arena); + return map; +} + +void PyUpb_WeakMap_Free(PyUpb_WeakMap* map) { upb_Arena_Free(map->arena); } + +// To give better entropy in the table key, we shift away low bits that are +// always zero. +static const int PyUpb_PtrShift = (sizeof(void*) == 4) ? 2 : 3; + +uintptr_t PyUpb_WeakMap_GetKey(const void* key) { + uintptr_t n = (uintptr_t)key; + assert((n & ((1 << PyUpb_PtrShift) - 1)) == 0); + return n >> PyUpb_PtrShift; +} + +void PyUpb_WeakMap_Add(PyUpb_WeakMap* map, const void* key, PyObject* py_obj) { + upb_inttable_insert(&map->table, PyUpb_WeakMap_GetKey(key), + upb_value_ptr(py_obj), map->arena); +} + +void PyUpb_WeakMap_Delete(PyUpb_WeakMap* map, const void* key) { + upb_value val; + bool removed = + upb_inttable_remove(&map->table, PyUpb_WeakMap_GetKey(key), &val); + (void)removed; + assert(removed); +} + +void PyUpb_WeakMap_TryDelete(PyUpb_WeakMap* map, const void* key) { + upb_inttable_remove(&map->table, PyUpb_WeakMap_GetKey(key), NULL); +} + +PyObject* PyUpb_WeakMap_Get(PyUpb_WeakMap* map, const void* key) { + upb_value val; + if (upb_inttable_lookup(&map->table, PyUpb_WeakMap_GetKey(key), &val)) { + PyObject* ret = upb_value_getptr(val); + Py_INCREF(ret); + return ret; + } else { + return NULL; + } +} + +bool PyUpb_WeakMap_Next(PyUpb_WeakMap* map, const void** key, PyObject** obj, + intptr_t* iter) { + uintptr_t u_key; + upb_value val; + if (!upb_inttable_next(&map->table, &u_key, &val, iter)) return false; + *key = (void*)(u_key << PyUpb_PtrShift); + *obj = upb_value_getptr(val); + return true; +} + +void PyUpb_WeakMap_DeleteIter(PyUpb_WeakMap* map, intptr_t* iter) { + upb_inttable_removeiter(&map->table, iter); +} + +// ----------------------------------------------------------------------------- +// ObjCache +// ----------------------------------------------------------------------------- + +PyUpb_WeakMap* PyUpb_ObjCache_Instance(void) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + return state->obj_cache; +} + +void PyUpb_ObjCache_Add(const void* key, PyObject* py_obj) { + PyUpb_WeakMap_Add(PyUpb_ObjCache_Instance(), key, py_obj); +} + +void PyUpb_ObjCache_Delete(const void* key) { + PyUpb_ModuleState* state = PyUpb_ModuleState_MaybeGet(); + if (!state) { + // During the shutdown sequence, our object's Dealloc() methods can be + // called *after* our module Dealloc() method has been called. At that + // point our state will be NULL and there is nothing to delete out of the + // map. + return; + } + PyUpb_WeakMap_Delete(state->obj_cache, key); +} + +PyObject* PyUpb_ObjCache_Get(const void* key) { + return PyUpb_WeakMap_Get(PyUpb_ObjCache_Instance(), key); +} + +// ----------------------------------------------------------------------------- +// Arena +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + upb_Arena* arena; +} PyUpb_Arena; + +// begin:google_only +// static upb_alloc* global_alloc = &upb_alloc_global; +// end:google_only + +// begin:github_only +#ifdef __GLIBC__ +#include <malloc.h> // malloc_trim() +#endif + +// A special allocator that calls malloc_trim() periodically to release +// memory to the OS. Without this call, we appear to leak memory, at least +// as measured in RSS. +// +// We opt not to use this instead of PyMalloc (which would also solve the +// problem) because the latter requires the GIL to be held. This would make +// our messages unsafe to share with other languages that could free at +// unpredictable +// times. +static void* upb_trim_allocfunc(upb_alloc* alloc, void* ptr, size_t oldsize, + size_t size) { + (void)alloc; + (void)oldsize; + if (size == 0) { + free(ptr); +#ifdef __GLIBC__ + static int count = 0; + if (++count == 10000) { + malloc_trim(0); + count = 0; + } +#endif + return NULL; + } else { + return realloc(ptr, size); + } +} +static upb_alloc trim_alloc = {&upb_trim_allocfunc}; +static const upb_alloc* global_alloc = &trim_alloc; +// end:github_only + +static upb_Arena* PyUpb_NewArena(void) { + return upb_Arena_Init(NULL, 0, global_alloc); +} + +PyObject* PyUpb_Arena_New(void) { + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + PyUpb_Arena* arena = (void*)PyType_GenericAlloc(state->arena_type, 0); + arena->arena = PyUpb_NewArena(); + return &arena->ob_base; +} + +static void PyUpb_Arena_Dealloc(PyObject* self) { + upb_Arena_Free(PyUpb_Arena_Get(self)); + PyUpb_Dealloc(self); +} + +upb_Arena* PyUpb_Arena_Get(PyObject* arena) { + return ((PyUpb_Arena*)arena)->arena; +} + +static PyType_Slot PyUpb_Arena_Slots[] = { + {Py_tp_dealloc, PyUpb_Arena_Dealloc}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_Arena_Spec = { + PYUPB_MODULE_NAME ".Arena", + sizeof(PyUpb_Arena), + 0, // itemsize + Py_TPFLAGS_DEFAULT, + PyUpb_Arena_Slots, +}; + +static bool PyUpb_InitArena(PyObject* m) { + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + state->arena_type = PyUpb_AddClass(m, &PyUpb_Arena_Spec); + return state->arena_type; +} + +// ----------------------------------------------------------------------------- +// Utilities +// ----------------------------------------------------------------------------- + +PyTypeObject* AddObject(PyObject* m, const char* name, PyType_Spec* spec) { + PyObject* type = PyType_FromSpec(spec); + return type && PyModule_AddObject(m, name, type) == 0 ? (PyTypeObject*)type + : NULL; +} + +static const char* PyUpb_GetClassName(PyType_Spec* spec) { + // spec->name contains a fully-qualified name, like: + // google.protobuf.pyext._message.FooBar + // + // Find the rightmost '.' to get "FooBar". + const char* name = strrchr(spec->name, '.'); + assert(name); + return name + 1; +} + +PyTypeObject* PyUpb_AddClass(PyObject* m, PyType_Spec* spec) { + PyObject* type = PyType_FromSpec(spec); + const char* name = PyUpb_GetClassName(spec); + if (PyModule_AddObject(m, name, type) < 0) { + Py_XDECREF(type); + return NULL; + } + return (PyTypeObject*)type; +} + +PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec, + PyObject* bases) { + PyObject* type = PyType_FromSpecWithBases(spec, bases); + const char* name = PyUpb_GetClassName(spec); + if (PyModule_AddObject(m, name, type) < 0) { + Py_XDECREF(type); + return NULL; + } + return (PyTypeObject*)type; +} + +const char* PyUpb_GetStrData(PyObject* obj) { + if (PyUnicode_Check(obj)) { + return PyUnicode_AsUTF8AndSize(obj, NULL); + } else if (PyBytes_Check(obj)) { + return PyBytes_AsString(obj); + } else { + return NULL; + } +} + +const char* PyUpb_VerifyStrData(PyObject* obj) { + const char* ret = PyUpb_GetStrData(obj); + if (ret) return ret; + PyErr_Format(PyExc_TypeError, "Expected string: %S", obj); + return NULL; +} + +PyObject* PyUpb_Forbidden_New(PyObject* cls, PyObject* args, PyObject* kwds) { + PyObject* name = PyObject_GetAttrString(cls, "__name__"); + PyErr_Format(PyExc_RuntimeError, + "Objects of type %U may not be created directly.", name); + Py_XDECREF(name); + return NULL; +} + +bool PyUpb_IndexToRange(PyObject* index, Py_ssize_t size, Py_ssize_t* i, + Py_ssize_t* count, Py_ssize_t* step) { + assert(i && count && step); + if (PySlice_Check(index)) { + Py_ssize_t start, stop; + if (PySlice_Unpack(index, &start, &stop, step) < 0) return false; + *count = PySlice_AdjustIndices(size, &start, &stop, *step); + *i = start; + } else { + *i = PyNumber_AsSsize_t(index, PyExc_IndexError); + + if (*i == -1 && PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, "list indices must be integers"); + return false; + } + + if (*i < 0) *i += size; + *step = 0; + *count = 1; + + if (*i < 0 || size <= *i) { + PyErr_Format(PyExc_IndexError, "list index out of range"); + return false; + } + } + return true; +} + +// ----------------------------------------------------------------------------- +// Module Entry Point +// ----------------------------------------------------------------------------- + +__attribute__((visibility("default"))) PyMODINIT_FUNC PyInit__message(void) { + PyObject* m = PyModule_Create(&module_def); + if (!m) return NULL; + + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + + state->allow_oversize_protos = false; + state->wkt_bases = NULL; + state->obj_cache = PyUpb_WeakMap_New(); + state->c_descriptor_symtab = NULL; + + if (!PyUpb_InitDescriptorContainers(m) || !PyUpb_InitDescriptorPool(m) || + !PyUpb_InitDescriptor(m) || !PyUpb_InitArena(m) || + !PyUpb_InitExtensionDict(m) || !PyUpb_Map_Init(m) || + !PyUpb_InitMessage(m) || !PyUpb_Repeated_Init(m) || + !PyUpb_UnknownFields_Init(m)) { + Py_DECREF(m); + return NULL; + } + + // Temporary: an cookie we can use in the tests to ensure we are testing upb + // and not another protobuf library on the system. + PyModule_AddIntConstant(m, "_IS_UPB", 1); + + return m; +}
diff --git a/python/protobuf.h b/python/protobuf.h new file mode 100644 index 0000000..e9839be --- /dev/null +++ b/python/protobuf.h
@@ -0,0 +1,240 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_PROTOBUF_H__ +#define PYUPB_PROTOBUF_H__ + +#include <stdbool.h> + +#include "python/descriptor.h" +#include "python/python_api.h" +#include "upb/hash/int_table.h" + +// begin:github_only +#define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google.protobuf" +#define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google.protobuf.internal" +#define PYUPB_DESCRIPTOR_PROTO_PACKAGE "google.protobuf" +#define PYUPB_MODULE_NAME "google._upb._message" +// end:github_only + +// begin:google_only +// #define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google3.net.google.protobuf.python.public" +// #define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google3.net.google.protobuf.python.internal" +// #define PYUPB_DESCRIPTOR_PROTO_PACKAGE "proto2" +// #define PYUPB_MODULE_NAME "google3.third_party.upb.python._message" +// end:google_only + +#define PYUPB_DESCRIPTOR_MODULE "google.protobuf.descriptor_pb2" +#define PYUPB_RETURN_OOM return PyErr_SetNone(PyExc_MemoryError), NULL + +struct PyUpb_WeakMap; +typedef struct PyUpb_WeakMap PyUpb_WeakMap; + +// ----------------------------------------------------------------------------- +// ModuleState +// ----------------------------------------------------------------------------- + +// We store all "global" state in this struct instead of using (C) global +// variables. This makes this extension compatible with sub-interpreters. + +typedef struct { + // From descriptor.c + PyTypeObject* descriptor_types[kPyUpb_Descriptor_Count]; + + // From descriptor_containers.c + PyTypeObject* by_name_map_type; + PyTypeObject* by_name_iterator_type; + PyTypeObject* by_number_map_type; + PyTypeObject* by_number_iterator_type; + PyTypeObject* generic_sequence_type; + + // From descriptor_pool.c + PyObject* default_pool; + + // From descriptor_pool.c + PyTypeObject* descriptor_pool_type; + upb_DefPool* c_descriptor_symtab; + + // From extension_dict.c + PyTypeObject* extension_dict_type; + PyTypeObject* extension_iterator_type; + + // From map.c + PyTypeObject* map_iterator_type; + PyTypeObject* message_map_container_type; + PyTypeObject* scalar_map_container_type; + + // From message.c + PyObject* decode_error_class; + PyObject* descriptor_string; + PyObject* encode_error_class; + PyObject* enum_type_wrapper_class; + PyObject* message_class; + PyTypeObject* cmessage_type; + PyTypeObject* message_meta_type; + PyObject* listfields_item_key; + + // From protobuf.c + bool allow_oversize_protos; + PyObject* wkt_bases; + PyTypeObject* arena_type; + PyUpb_WeakMap* obj_cache; + + // From repeated.c + PyTypeObject* repeated_composite_container_type; + PyTypeObject* repeated_scalar_container_type; + + // From unknown_fields.c + PyTypeObject* unknown_fields_type; + PyObject* unknown_field_type; +} PyUpb_ModuleState; + +// Returns the global state object from the current interpreter. The current +// interpreter is looked up from thread-local state. +PyUpb_ModuleState* PyUpb_ModuleState_Get(void); +PyUpb_ModuleState* PyUpb_ModuleState_GetFromModule(PyObject* module); + +// Returns NULL if module state is not yet available (during startup). +// Any use of the module state during startup needs to be passed explicitly. +PyUpb_ModuleState* PyUpb_ModuleState_MaybeGet(void); + +// Returns: +// from google.protobuf.internal.well_known_types import WKTBASES +// +// This has to be imported lazily rather than at module load time, because +// otherwise it would cause a circular import. +PyObject* PyUpb_GetWktBases(PyUpb_ModuleState* state); + +// ----------------------------------------------------------------------------- +// WeakMap +// ----------------------------------------------------------------------------- + +// A WeakMap maps C pointers to the corresponding Python wrapper object. We +// want a consistent Python wrapper object for each C object, both to save +// memory and to provide object stability (ie. x is x). +// +// Each wrapped object should add itself to the map when it is constructed and +// remove itself from the map when it is destroyed. The map is weak so it does +// not take references to the cached objects. + +PyUpb_WeakMap* PyUpb_WeakMap_New(void); +void PyUpb_WeakMap_Free(PyUpb_WeakMap* map); + +// Adds the given object to the map, indexed by the given key. +void PyUpb_WeakMap_Add(PyUpb_WeakMap* map, const void* key, PyObject* py_obj); + +// Removes the given key from the cache. It must exist in the cache currently. +void PyUpb_WeakMap_Delete(PyUpb_WeakMap* map, const void* key); +void PyUpb_WeakMap_TryDelete(PyUpb_WeakMap* map, const void* key); + +// Returns a new reference to an object if it exists, otherwise returns NULL. +PyObject* PyUpb_WeakMap_Get(PyUpb_WeakMap* map, const void* key); + +#define PYUPB_WEAKMAP_BEGIN UPB_INTTABLE_BEGIN + +// Iteration over the weak map, eg. +// +// intptr_t it = PYUPB_WEAKMAP_BEGIN; +// while (PyUpb_WeakMap_Next(map, &key, &obj, &it)) { +// // ... +// } +// +// Note that the callee does not own a ref on the returned `obj`. +bool PyUpb_WeakMap_Next(PyUpb_WeakMap* map, const void** key, PyObject** obj, + intptr_t* iter); +void PyUpb_WeakMap_DeleteIter(PyUpb_WeakMap* map, intptr_t* iter); + +// ----------------------------------------------------------------------------- +// ObjCache +// ----------------------------------------------------------------------------- + +// The object cache is a global WeakMap for mapping upb objects to the +// corresponding wrapper. +void PyUpb_ObjCache_Add(const void* key, PyObject* py_obj); +void PyUpb_ObjCache_Delete(const void* key); +PyObject* PyUpb_ObjCache_Get(const void* key); // returns NULL if not present. +PyUpb_WeakMap* PyUpb_ObjCache_Instance(void); + +// ----------------------------------------------------------------------------- +// Arena +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_Arena_New(void); +upb_Arena* PyUpb_Arena_Get(PyObject* arena); + +// ----------------------------------------------------------------------------- +// Utilities +// ----------------------------------------------------------------------------- + +PyTypeObject* AddObject(PyObject* m, const char* name, PyType_Spec* spec); + +// Creates a Python type from `spec` and adds it to the given module `m`. +PyTypeObject* PyUpb_AddClass(PyObject* m, PyType_Spec* spec); + +// Like PyUpb_AddClass(), but allows you to specify a tuple of base classes +// in `bases`. +PyTypeObject* PyUpb_AddClassWithBases(PyObject* m, PyType_Spec* spec, + PyObject* bases); + +// A function that implements the tp_new slot for types that we do not allow +// users to create directly. This will immediately fail with an error message. +PyObject* PyUpb_Forbidden_New(PyObject* cls, PyObject* args, PyObject* kwds); + +// Our standard dealloc func. It follows the guidance defined in: +// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc +// However it tests Py_TPFLAGS_HEAPTYPE dynamically so that a single dealloc +// function can work for any type. +static inline void PyUpb_Dealloc(void* self) { + PyTypeObject* tp = Py_TYPE(self); + assert(PyType_GetFlags(tp) & Py_TPFLAGS_HEAPTYPE); + freefunc tp_free = (freefunc)PyType_GetSlot(tp, Py_tp_free); + tp_free(self); + Py_DECREF(tp); +} + +// Equivalent to the Py_NewRef() function introduced in Python 3.10. If/when we +// drop support for Python <3.10, we can remove this function and replace all +// callers with Py_NewRef(). +static inline PyObject* PyUpb_NewRef(PyObject* obj) { + Py_INCREF(obj); + return obj; +} + +const char* PyUpb_GetStrData(PyObject* obj); +const char* PyUpb_VerifyStrData(PyObject* obj); + +// For an expression like: +// foo[index] +// +// Converts `index` to an effective i/count/step, for a repeated field +// or descriptor sequence of size 'size'. +bool PyUpb_IndexToRange(PyObject* index, Py_ssize_t size, Py_ssize_t* i, + Py_ssize_t* count, Py_ssize_t* step); +#endif // PYUPB_PROTOBUF_H__
diff --git a/python/py_extension.bzl b/python/py_extension.bzl new file mode 100644 index 0000000..7b918bc --- /dev/null +++ b/python/py_extension.bzl
@@ -0,0 +1,60 @@ +"""Macro to support py_extension """ + +load("@bazel_skylib//lib:selects.bzl", "selects") + +def py_extension(name, srcs, copts, deps = [], **kwargs): + """Creates a C++ library to extend python + + Args: + name: Name of the target + srcs: List of source files to create the target + copts: List of C++ compile options to use + deps: Libraries that the target depends on + """ + + native.cc_binary( + name = name + "_binary", + srcs = srcs, + copts = copts + ["-fvisibility=hidden"], + linkopts = selects.with_or({ + ( + "//python/dist:osx_x86_64", + "//python/dist:osx_aarch64", + ): ["-undefined", "dynamic_lookup"], + "//python/dist:windows_x86_32": ["-static-libgcc"], + "//conditions:default": [], + }), + linkshared = True, + linkstatic = True, + deps = deps + select({ + "//python:limited_api_3.7": ["@python-3.7.0//:python_headers"], + "//python:full_api_3.7_win32": ["@nuget_python_i686_3.7.0//:python_full_api"], + "//python:full_api_3.7_win64": ["@nuget_python_x86-64_3.7.0//:python_full_api"], + "//python:full_api_3.8_win32": ["@nuget_python_i686_3.8.0//:python_full_api"], + "//python:full_api_3.8_win64": ["@nuget_python_x86-64_3.8.0//:python_full_api"], + "//python:full_api_3.9_win32": ["@nuget_python_i686_3.9.0//:python_full_api"], + "//python:full_api_3.9_win64": ["@nuget_python_x86-64_3.9.0//:python_full_api"], + "//python:limited_api_3.10_win32": ["@nuget_python_i686_3.10.0//:python_limited_api"], + "//python:limited_api_3.10_win64": ["@nuget_python_x86-64_3.10.0//:python_limited_api"], + "//conditions:default": ["@system_python//:python_headers"], + }), + **kwargs + ) + + EXT_SUFFIX = ".abi3.so" + output_file = "google/_upb/" + name + EXT_SUFFIX + + native.genrule( + name = "copy" + name, + srcs = [":" + name + "_binary"], + outs = [output_file], + cmd = "cp $< $@", + visibility = ["//python:__subpackages__"], + ) + + native.py_library( + name = name, + data = [output_file], + imports = ["."], + visibility = ["//python:__subpackages__"], + )
diff --git a/python/python_api.h b/python/python_api.h new file mode 100644 index 0000000..fae7df2 --- /dev/null +++ b/python/python_api.h
@@ -0,0 +1,64 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_PYTHON_H__ +#define PYUPB_PYTHON_H__ + +// We restrict ourselves to the limited API, so that a single build can be +// ABI-compatible with a wide range of Python versions. +// +// The build system will define Py_LIMITED_API as appropriate (see BUILD). We +// only want to define it for our distribution packages, since we can do some +// extra assertions when Py_LIMITED_API is not defined. Also Py_LIMITED_API is +// incompatible with Py_DEBUG. + +// #define Py_LIMITED_API <val> // Defined by build system when appropriate. + +#include "Python.h" + +// Ideally we could restrict ourselves to the limited API of 3.7, but this is +// a very important function that was not officially added to the limited API +// until 3.10. Without this function, there is no way of getting data from a +// Python `str` object without a copy. +// +// While this function was not *officially* added to the limited API until +// Python 3.10, In practice it has been stable since Python 3.1. +// https://bugs.python.org/issue41784 +// +// On Linux/ELF and macOS/Mach-O, we can get away with using this function with +// the limited API prior to 3.10. + +#if (defined(__linux__) || defined(__APPLE__)) && defined(Py_LIMITED_API) && \ + Py_LIMITED_API < 0x03100000 +PyAPI_FUNC(const char*) + PyUnicode_AsUTF8AndSize(PyObject* unicode, Py_ssize_t* size); +#endif + +#endif // PYUPB_PYTHON_H__
diff --git a/python/repeated.c b/python/repeated.c new file mode 100644 index 0000000..abb34e8 --- /dev/null +++ b/python/repeated.c
@@ -0,0 +1,800 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/repeated.h" + +#include "python/convert.h" +#include "python/message.h" +#include "python/protobuf.h" + +static PyObject* PyUpb_RepeatedCompositeContainer_Append(PyObject* _self, + PyObject* value); +static PyObject* PyUpb_RepeatedScalarContainer_Append(PyObject* _self, + PyObject* value); + +// Wrapper for a repeated field. +typedef struct { + PyObject_HEAD; + PyObject* arena; + // The field descriptor (PyObject*). + // The low bit indicates whether the container is reified (see ptr below). + // - low bit set: repeated field is a stub (no underlying data). + // - low bit clear: repeated field is reified (points to upb_Array). + uintptr_t field; + union { + PyObject* parent; // stub: owning pointer to parent message. + upb_Array* arr; // reified: the data for this array. + } ptr; +} PyUpb_RepeatedContainer; + +static bool PyUpb_RepeatedContainer_IsStub(PyUpb_RepeatedContainer* self) { + return self->field & 1; +} + +static PyObject* PyUpb_RepeatedContainer_GetFieldDescriptor( + PyUpb_RepeatedContainer* self) { + return (PyObject*)(self->field & ~(uintptr_t)1); +} + +static const upb_FieldDef* PyUpb_RepeatedContainer_GetField( + PyUpb_RepeatedContainer* self) { + return PyUpb_FieldDescriptor_GetDef( + PyUpb_RepeatedContainer_GetFieldDescriptor(self)); +} + +// If the repeated field is reified, returns it. Otherwise, returns NULL. +// If NULL is returned, the object is empty and has no underlying data. +static upb_Array* PyUpb_RepeatedContainer_GetIfReified( + PyUpb_RepeatedContainer* self) { + return PyUpb_RepeatedContainer_IsStub(self) ? NULL : self->ptr.arr; +} + +void PyUpb_RepeatedContainer_Reify(PyObject* _self, upb_Array* arr) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + assert(PyUpb_RepeatedContainer_IsStub(self)); + if (!arr) { + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + arr = upb_Array_New(arena, upb_FieldDef_CType(f)); + } + PyUpb_ObjCache_Add(arr, &self->ob_base); + Py_DECREF(self->ptr.parent); + self->ptr.arr = arr; // Overwrites self->ptr.parent. + self->field &= ~(uintptr_t)1; + assert(!PyUpb_RepeatedContainer_IsStub(self)); +} + +upb_Array* PyUpb_RepeatedContainer_EnsureReified(PyObject* _self) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_GetIfReified(self); + if (arr) return arr; // Already writable. + + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + arr = upb_Array_New(arena, upb_FieldDef_CType(f)); + PyUpb_Message_SetConcreteSubobj(self->ptr.parent, f, + (upb_MessageValue){.array_val = arr}); + PyUpb_RepeatedContainer_Reify((PyObject*)self, arr); + return arr; +} + +static void PyUpb_RepeatedContainer_Dealloc(PyObject* _self) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + Py_DECREF(self->arena); + if (PyUpb_RepeatedContainer_IsStub(self)) { + PyUpb_Message_CacheDelete(self->ptr.parent, + PyUpb_RepeatedContainer_GetField(self)); + Py_DECREF(self->ptr.parent); + } else { + PyUpb_ObjCache_Delete(self->ptr.arr); + } + Py_DECREF(PyUpb_RepeatedContainer_GetFieldDescriptor(self)); + PyUpb_Dealloc(self); +} + +static PyTypeObject* PyUpb_RepeatedContainer_GetClass(const upb_FieldDef* f) { + assert(upb_FieldDef_IsRepeated(f) && !upb_FieldDef_IsMap(f)); + PyUpb_ModuleState* state = PyUpb_ModuleState_Get(); + return upb_FieldDef_IsSubMessage(f) ? state->repeated_composite_container_type + : state->repeated_scalar_container_type; +} + +static Py_ssize_t PyUpb_RepeatedContainer_Length(PyObject* self) { + upb_Array* arr = + PyUpb_RepeatedContainer_GetIfReified((PyUpb_RepeatedContainer*)self); + return arr ? upb_Array_Size(arr) : 0; +} + +PyObject* PyUpb_RepeatedContainer_NewStub(PyObject* parent, + const upb_FieldDef* f, + PyObject* arena) { + // We only create stubs when the parent is reified, by convention. However + // this is not an invariant: the parent could become reified at any time. + assert(PyUpb_Message_GetIfReified(parent) == NULL); + PyTypeObject* cls = PyUpb_RepeatedContainer_GetClass(f); + PyUpb_RepeatedContainer* repeated = (void*)PyType_GenericAlloc(cls, 0); + repeated->arena = arena; + repeated->field = (uintptr_t)PyUpb_FieldDescriptor_Get(f) | 1; + repeated->ptr.parent = parent; + Py_INCREF(arena); + Py_INCREF(parent); + return &repeated->ob_base; +} + +PyObject* PyUpb_RepeatedContainer_GetOrCreateWrapper(upb_Array* arr, + const upb_FieldDef* f, + PyObject* arena) { + PyObject* ret = PyUpb_ObjCache_Get(arr); + if (ret) return ret; + + PyTypeObject* cls = PyUpb_RepeatedContainer_GetClass(f); + PyUpb_RepeatedContainer* repeated = (void*)PyType_GenericAlloc(cls, 0); + repeated->arena = arena; + repeated->field = (uintptr_t)PyUpb_FieldDescriptor_Get(f); + repeated->ptr.arr = arr; + ret = &repeated->ob_base; + Py_INCREF(arena); + PyUpb_ObjCache_Add(arr, ret); + return ret; +} + +static PyObject* PyUpb_RepeatedContainer_MergeFrom(PyObject* _self, + PyObject* args); + +PyObject* PyUpb_RepeatedContainer_DeepCopy(PyObject* _self, PyObject* value) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + PyUpb_RepeatedContainer* clone = + (void*)PyType_GenericAlloc(Py_TYPE(_self), 0); + if (clone == NULL) return NULL; + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + clone->arena = PyUpb_Arena_New(); + clone->field = (uintptr_t)PyUpb_FieldDescriptor_Get(f); + clone->ptr.arr = + upb_Array_New(PyUpb_Arena_Get(clone->arena), upb_FieldDef_CType(f)); + PyUpb_ObjCache_Add(clone->ptr.arr, (PyObject*)clone); + PyObject* result = PyUpb_RepeatedContainer_MergeFrom((PyObject*)clone, _self); + if (!result) { + Py_DECREF(clone); + return NULL; + } + Py_DECREF(result); + return (PyObject*)clone; +} + +PyObject* PyUpb_RepeatedContainer_Extend(PyObject* _self, PyObject* value) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + size_t start_size = upb_Array_Size(arr); + PyObject* it = PyObject_GetIter(value); + if (!it) { + PyErr_SetString(PyExc_TypeError, "Value must be iterable"); + return NULL; + } + + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + bool submsg = upb_FieldDef_IsSubMessage(f); + PyObject* e; + + while ((e = PyIter_Next(it))) { + PyObject* ret; + if (submsg) { + ret = PyUpb_RepeatedCompositeContainer_Append(_self, e); + } else { + ret = PyUpb_RepeatedScalarContainer_Append(_self, e); + } + Py_XDECREF(ret); + Py_DECREF(e); + } + + Py_DECREF(it); + + if (PyErr_Occurred()) { + upb_Array_Resize(arr, start_size, NULL); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* PyUpb_RepeatedContainer_Item(PyObject* _self, + Py_ssize_t index) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_GetIfReified(self); + Py_ssize_t size = arr ? upb_Array_Size(arr) : 0; + if (index < 0 || index >= size) { + PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index); + return NULL; + } + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + return PyUpb_UpbToPy(upb_Array_Get(arr, index), f, self->arena); +} + +PyObject* PyUpb_RepeatedContainer_ToList(PyObject* _self) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_GetIfReified(self); + if (!arr) return PyList_New(0); + + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + size_t n = upb_Array_Size(arr); + PyObject* list = PyList_New(n); + for (size_t i = 0; i < n; i++) { + PyObject* val = PyUpb_UpbToPy(upb_Array_Get(arr, i), f, self->arena); + if (!val) { + Py_DECREF(list); + return NULL; + } + PyList_SetItem(list, i, val); + } + return list; +} + +static PyObject* PyUpb_RepeatedContainer_Repr(PyObject* _self) { + PyObject* list = PyUpb_RepeatedContainer_ToList(_self); + if (!list) return NULL; + assert(!PyErr_Occurred()); + PyObject* repr = PyObject_Repr(list); + Py_DECREF(list); + return repr; +} + +static PyObject* PyUpb_RepeatedContainer_RichCompare(PyObject* _self, + PyObject* _other, + int opid) { + if (opid != Py_EQ && opid != Py_NE) { + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + PyObject* list1 = PyUpb_RepeatedContainer_ToList(_self); + PyObject* list2 = _other; + PyObject* del = NULL; + if (PyObject_TypeCheck(_other, _self->ob_type)) { + del = list2 = PyUpb_RepeatedContainer_ToList(_other); + } + PyObject* ret = PyObject_RichCompare(list1, list2, opid); + Py_DECREF(list1); + Py_XDECREF(del); + return ret; +} + +static PyObject* PyUpb_RepeatedContainer_Subscript(PyObject* _self, + PyObject* key) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_GetIfReified(self); + Py_ssize_t size = arr ? upb_Array_Size(arr) : 0; + Py_ssize_t idx, count, step; + if (!PyUpb_IndexToRange(key, size, &idx, &count, &step)) return NULL; + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + if (step == 0) { + return PyUpb_UpbToPy(upb_Array_Get(arr, idx), f, self->arena); + } else { + PyObject* list = PyList_New(count); + for (Py_ssize_t i = 0; i < count; i++, idx += step) { + upb_MessageValue msgval = upb_Array_Get(self->ptr.arr, idx); + PyObject* item = PyUpb_UpbToPy(msgval, f, self->arena); + if (!item) { + Py_DECREF(list); + return NULL; + } + PyList_SetItem(list, i, item); + } + return list; + } +} + +static int PyUpb_RepeatedContainer_SetSubscript( + PyUpb_RepeatedContainer* self, upb_Array* arr, const upb_FieldDef* f, + Py_ssize_t idx, Py_ssize_t count, Py_ssize_t step, PyObject* value) { + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + if (upb_FieldDef_IsSubMessage(f)) { + PyErr_SetString(PyExc_TypeError, "does not support assignment"); + return -1; + } + + if (step == 0) { + // Set single value. + upb_MessageValue msgval; + if (!PyUpb_PyToUpb(value, f, &msgval, arena)) return -1; + upb_Array_Set(arr, idx, msgval); + return 0; + } + + // Set range. + PyObject* seq = + PySequence_Fast(value, "must assign iterable to extended slice"); + PyObject* item = NULL; + int ret = -1; + if (!seq) goto err; + Py_ssize_t seq_size = PySequence_Size(seq); + if (seq_size != count) { + if (step == 1) { + // We must shift the tail elements (either right or left). + size_t tail = upb_Array_Size(arr) - (idx + count); + upb_Array_Resize(arr, idx + seq_size + tail, arena); + upb_Array_Move(arr, idx + seq_size, idx + count, tail); + count = seq_size; + } else { + PyErr_Format(PyExc_ValueError, + "attempt to assign sequence of %zd to extended slice " + "of size %zd", + seq_size, count); + goto err; + } + } + for (Py_ssize_t i = 0; i < count; i++, idx += step) { + upb_MessageValue msgval; + item = PySequence_GetItem(seq, i); + if (!item) goto err; + // XXX: if this fails we can leave the list partially mutated. + if (!PyUpb_PyToUpb(item, f, &msgval, arena)) goto err; + Py_DECREF(item); + item = NULL; + upb_Array_Set(arr, idx, msgval); + } + ret = 0; + +err: + Py_XDECREF(seq); + Py_XDECREF(item); + return ret; +} + +static int PyUpb_RepeatedContainer_DeleteSubscript(upb_Array* arr, + Py_ssize_t idx, + Py_ssize_t count, + Py_ssize_t step) { + // Normalize direction: deletion is order-independent. + Py_ssize_t start = idx; + if (step < 0) { + Py_ssize_t end = start + step * (count - 1); + start = end; + step = -step; + } + + size_t dst = start; + size_t src; + if (step > 1) { + // Move elements between steps: + // + // src + // | + // |------X---X---X---X------------------------------| + // | + // dst <-------- tail --------------> + src = start + 1; + for (Py_ssize_t i = 1; i < count; i++, dst += step - 1, src += step) { + upb_Array_Move(arr, dst, src, step); + } + } else { + src = start + count; + } + + // Move tail. + size_t tail = upb_Array_Size(arr) - src; + size_t new_size = dst + tail; + assert(new_size == upb_Array_Size(arr) - count); + upb_Array_Move(arr, dst, src, tail); + upb_Array_Resize(arr, new_size, NULL); + return 0; +} + +static int PyUpb_RepeatedContainer_AssignSubscript(PyObject* _self, + PyObject* key, + PyObject* value) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + Py_ssize_t size = arr ? upb_Array_Size(arr) : 0; + Py_ssize_t idx, count, step; + if (!PyUpb_IndexToRange(key, size, &idx, &count, &step)) return -1; + if (value) { + return PyUpb_RepeatedContainer_SetSubscript(self, arr, f, idx, count, step, + value); + } else { + return PyUpb_RepeatedContainer_DeleteSubscript(arr, idx, count, step); + } +} + +static PyObject* PyUpb_RepeatedContainer_Pop(PyObject* _self, PyObject* args) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n", &index)) return NULL; + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + size_t size = upb_Array_Size(arr); + if (index < 0) index += size; + if (index >= size) index = size - 1; + PyObject* ret = PyUpb_RepeatedContainer_Item(_self, index); + if (!ret) return NULL; + upb_Array_Delete(self->ptr.arr, index, 1); + return ret; +} + +static PyObject* PyUpb_RepeatedContainer_Remove(PyObject* _self, + PyObject* value) { + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + Py_ssize_t match_index = -1; + Py_ssize_t n = PyUpb_RepeatedContainer_Length(_self); + for (Py_ssize_t i = 0; i < n; ++i) { + PyObject* elem = PyUpb_RepeatedContainer_Item(_self, i); + if (!elem) return NULL; + int eq = PyObject_RichCompareBool(elem, value, Py_EQ); + Py_DECREF(elem); + if (eq) { + match_index = i; + break; + } + } + if (match_index == -1) { + PyErr_SetString(PyExc_ValueError, "remove(x): x not in container"); + return NULL; + } + if (PyUpb_RepeatedContainer_DeleteSubscript(arr, match_index, 1, 1) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +// A helper function used only for Sort(). +static bool PyUpb_RepeatedContainer_Assign(PyObject* _self, PyObject* list) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + Py_ssize_t size = PyList_Size(list); + bool submsg = upb_FieldDef_IsSubMessage(f); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + for (Py_ssize_t i = 0; i < size; ++i) { + PyObject* obj = PyList_GetItem(list, i); + upb_MessageValue msgval; + if (submsg) { + msgval.msg_val = PyUpb_Message_GetIfReified(obj); + assert(msgval.msg_val); + } else { + if (!PyUpb_PyToUpb(obj, f, &msgval, arena)) return false; + } + upb_Array_Set(arr, i, msgval); + } + return true; +} + +static PyObject* PyUpb_RepeatedContainer_Sort(PyObject* pself, PyObject* args, + PyObject* kwds) { + // Support the old sort_function argument for backwards + // compatibility. + if (kwds != NULL) { + PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function"); + if (sort_func != NULL) { + // Must set before deleting as sort_func is a borrowed reference + // and kwds might be the only thing keeping it alive. + if (PyDict_SetItemString(kwds, "cmp", sort_func) == -1) return NULL; + if (PyDict_DelItemString(kwds, "sort_function") == -1) return NULL; + } + } + + PyObject* ret = NULL; + PyObject* full_slice = NULL; + PyObject* list = NULL; + PyObject* m = NULL; + PyObject* res = NULL; + if ((full_slice = PySlice_New(NULL, NULL, NULL)) && + (list = PyUpb_RepeatedContainer_Subscript(pself, full_slice)) && + (m = PyObject_GetAttrString(list, "sort")) && + (res = PyObject_Call(m, args, kwds)) && + PyUpb_RepeatedContainer_Assign(pself, list)) { + Py_INCREF(Py_None); + ret = Py_None; + } + + Py_XDECREF(full_slice); + Py_XDECREF(list); + Py_XDECREF(m); + Py_XDECREF(res); + return ret; +} + +static PyObject* PyUpb_RepeatedContainer_Reverse(PyObject* _self) { + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + size_t n = upb_Array_Size(arr); + size_t half = n / 2; // Rounds down. + for (size_t i = 0; i < half; i++) { + size_t i2 = n - i - 1; + upb_MessageValue val1 = upb_Array_Get(arr, i); + upb_MessageValue val2 = upb_Array_Get(arr, i2); + upb_Array_Set(arr, i, val2); + upb_Array_Set(arr, i2, val1); + } + Py_RETURN_NONE; +} + +static PyObject* PyUpb_RepeatedContainer_MergeFrom(PyObject* _self, + PyObject* args) { + return PyUpb_RepeatedContainer_Extend(_self, args); +} + +// ----------------------------------------------------------------------------- +// RepeatedCompositeContainer +// ----------------------------------------------------------------------------- + +static PyObject* PyUpb_RepeatedCompositeContainer_AppendNew(PyObject* _self) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + if (!arr) return NULL; + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + const upb_MessageDef* m = upb_FieldDef_MessageSubDef(f); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + upb_Message* msg = upb_Message_New(layout, arena); + upb_MessageValue msgval = {.msg_val = msg}; + upb_Array_Append(arr, msgval, arena); + return PyUpb_Message_Get(msg, m, self->arena); +} + +PyObject* PyUpb_RepeatedCompositeContainer_Add(PyObject* _self, PyObject* args, + PyObject* kwargs) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + PyObject* py_msg = PyUpb_RepeatedCompositeContainer_AppendNew(_self); + if (!py_msg) return NULL; + if (PyUpb_Message_InitAttributes(py_msg, args, kwargs) < 0) { + Py_DECREF(py_msg); + upb_Array_Delete(self->ptr.arr, upb_Array_Size(self->ptr.arr) - 1, 1); + return NULL; + } + return py_msg; +} + +static PyObject* PyUpb_RepeatedCompositeContainer_Append(PyObject* _self, + PyObject* value) { + if (!PyUpb_Message_Verify(value)) return NULL; + PyObject* py_msg = PyUpb_RepeatedCompositeContainer_AppendNew(_self); + if (!py_msg) return NULL; + PyObject* none = PyUpb_Message_MergeFrom(py_msg, value); + if (!none) { + Py_DECREF(py_msg); + return NULL; + } + Py_DECREF(none); + return py_msg; +} + +static PyObject* PyUpb_RepeatedContainer_Insert(PyObject* _self, + PyObject* args) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + Py_ssize_t index; + PyObject* value; + if (!PyArg_ParseTuple(args, "nO", &index, &value)) return NULL; + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + if (!arr) return NULL; + + // Normalize index. + Py_ssize_t size = upb_Array_Size(arr); + if (index < 0) index += size; + if (index < 0) index = 0; + if (index > size) index = size; + + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_MessageValue msgval; + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + if (upb_FieldDef_IsSubMessage(f)) { + // Create message. + const upb_MessageDef* m = upb_FieldDef_MessageSubDef(f); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + upb_Message* msg = upb_Message_New(layout, arena); + PyObject* py_msg = PyUpb_Message_Get(msg, m, self->arena); + PyObject* ret = PyUpb_Message_MergeFrom(py_msg, value); + Py_DECREF(py_msg); + if (!ret) return NULL; + Py_DECREF(ret); + msgval.msg_val = msg; + } else { + if (!PyUpb_PyToUpb(value, f, &msgval, arena)) return NULL; + } + + upb_Array_Insert(arr, index, 1, arena); + upb_Array_Set(arr, index, msgval); + + Py_RETURN_NONE; +} + +static PyMethodDef PyUpb_RepeatedCompositeContainer_Methods[] = { + {"__deepcopy__", PyUpb_RepeatedContainer_DeepCopy, METH_VARARGS, + "Makes a deep copy of the class."}, + {"add", (PyCFunction)PyUpb_RepeatedCompositeContainer_Add, + METH_VARARGS | METH_KEYWORDS, "Adds an object to the repeated container."}, + {"append", PyUpb_RepeatedCompositeContainer_Append, METH_O, + "Appends a message to the end of the repeated container."}, + {"insert", PyUpb_RepeatedContainer_Insert, METH_VARARGS, + "Inserts a message before the specified index."}, + {"extend", PyUpb_RepeatedContainer_Extend, METH_O, + "Adds objects to the repeated container."}, + {"pop", PyUpb_RepeatedContainer_Pop, METH_VARARGS, + "Removes an object from the repeated container and returns it."}, + {"remove", PyUpb_RepeatedContainer_Remove, METH_O, + "Removes an object from the repeated container."}, + {"sort", (PyCFunction)PyUpb_RepeatedContainer_Sort, + METH_VARARGS | METH_KEYWORDS, "Sorts the repeated container."}, + {"reverse", (PyCFunction)PyUpb_RepeatedContainer_Reverse, METH_NOARGS, + "Reverses elements order of the repeated container."}, + {"MergeFrom", PyUpb_RepeatedContainer_MergeFrom, METH_O, + "Adds objects to the repeated container."}, + {NULL, NULL}}; + +static PyType_Slot PyUpb_RepeatedCompositeContainer_Slots[] = { + {Py_tp_dealloc, PyUpb_RepeatedContainer_Dealloc}, + {Py_tp_methods, PyUpb_RepeatedCompositeContainer_Methods}, + {Py_sq_length, PyUpb_RepeatedContainer_Length}, + {Py_sq_item, PyUpb_RepeatedContainer_Item}, + {Py_mp_length, PyUpb_RepeatedContainer_Length}, + {Py_tp_repr, PyUpb_RepeatedContainer_Repr}, + {Py_mp_subscript, PyUpb_RepeatedContainer_Subscript}, + {Py_mp_ass_subscript, PyUpb_RepeatedContainer_AssignSubscript}, + {Py_tp_new, PyUpb_Forbidden_New}, + {Py_tp_richcompare, PyUpb_RepeatedContainer_RichCompare}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {0, NULL}}; + +static PyType_Spec PyUpb_RepeatedCompositeContainer_Spec = { + PYUPB_MODULE_NAME ".RepeatedCompositeContainer", + sizeof(PyUpb_RepeatedContainer), + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, + PyUpb_RepeatedCompositeContainer_Slots, +}; + +// ----------------------------------------------------------------------------- +// RepeatedScalarContainer +// ----------------------------------------------------------------------------- + +static PyObject* PyUpb_RepeatedScalarContainer_Append(PyObject* _self, + PyObject* value) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_EnsureReified(_self); + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_MessageValue msgval; + if (!PyUpb_PyToUpb(value, f, &msgval, arena)) { + return NULL; + } + upb_Array_Append(arr, msgval, arena); + Py_RETURN_NONE; +} + +static int PyUpb_RepeatedScalarContainer_AssignItem(PyObject* _self, + Py_ssize_t index, + PyObject* item) { + PyUpb_RepeatedContainer* self = (PyUpb_RepeatedContainer*)_self; + upb_Array* arr = PyUpb_RepeatedContainer_GetIfReified(self); + Py_ssize_t size = arr ? upb_Array_Size(arr) : 0; + if (index < 0 || index >= size) { + PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index); + return -1; + } + const upb_FieldDef* f = PyUpb_RepeatedContainer_GetField(self); + upb_MessageValue msgval; + upb_Arena* arena = PyUpb_Arena_Get(self->arena); + if (!PyUpb_PyToUpb(item, f, &msgval, arena)) { + return -1; + } + upb_Array_Set(self->ptr.arr, index, msgval); + return 0; +} + +static PyObject* PyUpb_RepeatedScalarContainer_Reduce(PyObject* unused_self, + PyObject* unused_other) { + PyObject* pickle_module = PyImport_ImportModule("pickle"); + if (!pickle_module) return NULL; + PyObject* pickle_error = PyObject_GetAttrString(pickle_module, "PickleError"); + Py_DECREF(pickle_module); + if (!pickle_error) return NULL; + PyErr_Format(pickle_error, + "can't pickle repeated message fields, convert to list first"); + Py_DECREF(pickle_error); + return NULL; +} + +static PyMethodDef PyUpb_RepeatedScalarContainer_Methods[] = { + {"__deepcopy__", PyUpb_RepeatedContainer_DeepCopy, METH_VARARGS, + "Makes a deep copy of the class."}, + {"__reduce__", PyUpb_RepeatedScalarContainer_Reduce, METH_NOARGS, + "Outputs picklable representation of the repeated field."}, + {"append", PyUpb_RepeatedScalarContainer_Append, METH_O, + "Appends an object to the repeated container."}, + {"extend", PyUpb_RepeatedContainer_Extend, METH_O, + "Appends objects to the repeated container."}, + {"insert", PyUpb_RepeatedContainer_Insert, METH_VARARGS, + "Inserts an object at the specified position in the container."}, + {"pop", PyUpb_RepeatedContainer_Pop, METH_VARARGS, + "Removes an object from the repeated container and returns it."}, + {"remove", PyUpb_RepeatedContainer_Remove, METH_O, + "Removes an object from the repeated container."}, + {"sort", (PyCFunction)PyUpb_RepeatedContainer_Sort, + METH_VARARGS | METH_KEYWORDS, "Sorts the repeated container."}, + {"reverse", (PyCFunction)PyUpb_RepeatedContainer_Reverse, METH_NOARGS, + "Reverses elements order of the repeated container."}, + {"MergeFrom", PyUpb_RepeatedContainer_MergeFrom, METH_O, + "Merges a repeated container into the current container."}, + {NULL, NULL}}; + +static PyType_Slot PyUpb_RepeatedScalarContainer_Slots[] = { + {Py_tp_dealloc, PyUpb_RepeatedContainer_Dealloc}, + {Py_tp_methods, PyUpb_RepeatedScalarContainer_Methods}, + {Py_tp_new, PyUpb_Forbidden_New}, + {Py_tp_repr, PyUpb_RepeatedContainer_Repr}, + {Py_sq_length, PyUpb_RepeatedContainer_Length}, + {Py_sq_item, PyUpb_RepeatedContainer_Item}, + {Py_sq_ass_item, PyUpb_RepeatedScalarContainer_AssignItem}, + {Py_mp_length, PyUpb_RepeatedContainer_Length}, + {Py_mp_subscript, PyUpb_RepeatedContainer_Subscript}, + {Py_mp_ass_subscript, PyUpb_RepeatedContainer_AssignSubscript}, + {Py_tp_richcompare, PyUpb_RepeatedContainer_RichCompare}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {0, NULL}}; + +static PyType_Spec PyUpb_RepeatedScalarContainer_Spec = { + PYUPB_MODULE_NAME ".RepeatedScalarContainer", + sizeof(PyUpb_RepeatedContainer), + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, + PyUpb_RepeatedScalarContainer_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +static bool PyUpb_Repeated_RegisterAsSequence(PyUpb_ModuleState* state) { + PyObject* collections = NULL; + PyObject* seq = NULL; + PyObject* ret1 = NULL; + PyObject* ret2 = NULL; + PyTypeObject* type1 = state->repeated_scalar_container_type; + PyTypeObject* type2 = state->repeated_composite_container_type; + + bool ok = (collections = PyImport_ImportModule("collections.abc")) && + (seq = PyObject_GetAttrString(collections, "MutableSequence")) && + (ret1 = PyObject_CallMethod(seq, "register", "O", type1)) && + (ret2 = PyObject_CallMethod(seq, "register", "O", type2)); + + Py_XDECREF(collections); + Py_XDECREF(seq); + Py_XDECREF(ret1); + Py_XDECREF(ret2); + return ok; +} + +bool PyUpb_Repeated_Init(PyObject* m) { + PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m); + + state->repeated_composite_container_type = + PyUpb_AddClass(m, &PyUpb_RepeatedCompositeContainer_Spec); + state->repeated_scalar_container_type = + PyUpb_AddClass(m, &PyUpb_RepeatedScalarContainer_Spec); + + return state->repeated_composite_container_type && + state->repeated_scalar_container_type && + PyUpb_Repeated_RegisterAsSequence(state); +}
diff --git a/python/repeated.h b/python/repeated.h new file mode 100644 index 0000000..54670e7 --- /dev/null +++ b/python/repeated.h
@@ -0,0 +1,72 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_REPEATED_H__ +#define PYUPB_REPEATED_H__ + +#include <stdbool.h> + +#include "python/python_api.h" +#include "upb/reflection/def.h" + +// Creates a new repeated field stub for field `f` of message object `parent`. +// Precondition: `parent` must be a stub. +PyObject* PyUpb_RepeatedContainer_NewStub(PyObject* parent, + const upb_FieldDef* f, + PyObject* arena); + +// Returns a repeated field object wrapping `arr`, of field type `f`, which +// must be on `arena`. If an existing wrapper object exists, it will be +// returned, otherwise a new object will be created. The caller always owns a +// ref on the returned value. +PyObject* PyUpb_RepeatedContainer_GetOrCreateWrapper(upb_Array* arr, + const upb_FieldDef* f, + PyObject* arena); + +// Reifies a repeated field stub to point to the concrete data in `arr`. +// If `arr` is NULL, an appropriate empty array will be constructed. +void PyUpb_RepeatedContainer_Reify(PyObject* self, upb_Array* arr); + +// Reifies this repeated object if it is not already reified. +upb_Array* PyUpb_RepeatedContainer_EnsureReified(PyObject* self); + +// Implements repeated_field.extend(iterable). `_self` must be a repeated +// field (either repeated composite or repeated scalar). +PyObject* PyUpb_RepeatedContainer_Extend(PyObject* _self, PyObject* value); + +// Implements repeated_field.add(initial_values). `_self` must be a repeated +// composite field. +PyObject* PyUpb_RepeatedCompositeContainer_Add(PyObject* _self, PyObject* args, + PyObject* kwargs); + +// Module-level init. +bool PyUpb_Repeated_Init(PyObject* m); + +#endif // PYUPB_REPEATED_H__
diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..ad71bf2 --- /dev/null +++ b/python/requirements.txt
@@ -0,0 +1 @@ +numpy<=1.24.4
diff --git a/python/unknown_fields.c b/python/unknown_fields.c new file mode 100644 index 0000000..f228f23 --- /dev/null +++ b/python/unknown_fields.c
@@ -0,0 +1,358 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "python/unknown_fields.h" + +#include "python/message.h" +#include "python/protobuf.h" +#include "upb/wire/eps_copy_input_stream.h" +#include "upb/wire/reader.h" +#include "upb/wire/types.h" + +// ----------------------------------------------------------------------------- +// UnknownFieldSet +// ----------------------------------------------------------------------------- + +typedef struct { + PyObject_HEAD; + PyObject* fields; +} PyUpb_UnknownFieldSet; + +static void PyUpb_UnknownFieldSet_Dealloc(PyObject* _self) { + PyUpb_UnknownFieldSet* self = (PyUpb_UnknownFieldSet*)_self; + Py_XDECREF(self->fields); + PyUpb_Dealloc(self); +} + +PyUpb_UnknownFieldSet* PyUpb_UnknownFieldSet_NewBare(void) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + PyUpb_UnknownFieldSet* self = + (void*)PyType_GenericAlloc(s->unknown_fields_type, 0); + return self; +} + +// For MessageSet the established behavior is for UnknownFieldSet to interpret +// the MessageSet wire format: +// message MessageSet { +// repeated group Item = 1 { +// required int32 type_id = 2; +// required bytes message = 3; +// } +// } +// +// And create unknown fields like: +// UnknownField(type_id, WIRE_TYPE_DELIMITED, message) +// +// For any unknown fields that are unexpected per the wire format defined above, +// we drop them on the floor. + +enum { + kUpb_MessageSet_StartItemTag = (1 << 3) | kUpb_WireType_StartGroup, + kUpb_MessageSet_EndItemTag = (1 << 3) | kUpb_WireType_EndGroup, + kUpb_MessageSet_TypeIdTag = (2 << 3) | kUpb_WireType_Varint, + kUpb_MessageSet_MessageTag = (3 << 3) | kUpb_WireType_Delimited, +}; + +static const char* PyUpb_UnknownFieldSet_BuildMessageSetItem( + PyUpb_UnknownFieldSet* self, upb_EpsCopyInputStream* stream, + const char* ptr) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + int type_id = 0; + PyObject* msg = NULL; + while (!upb_EpsCopyInputStream_IsDone(stream, &ptr)) { + uint32_t tag; + ptr = upb_WireReader_ReadTag(ptr, &tag); + if (!ptr) goto err; + switch (tag) { + case kUpb_MessageSet_EndItemTag: + goto done; + case kUpb_MessageSet_TypeIdTag: { + uint64_t tmp; + ptr = upb_WireReader_ReadVarint(ptr, &tmp); + if (!ptr) goto err; + if (!type_id) type_id = tmp; + break; + } + case kUpb_MessageSet_MessageTag: { + int size; + ptr = upb_WireReader_ReadSize(ptr, &size); + if (!upb_EpsCopyInputStream_CheckDataSizeAvailable(stream, ptr, size)) { + goto err; + } + const char* str = ptr; + ptr = upb_EpsCopyInputStream_ReadStringAliased(stream, &str, size); + if (!msg) { + msg = PyBytes_FromStringAndSize(str, size); + if (!msg) goto err; + } else { + // already saw a message here so deliberately skipping the duplicate + } + break; + } + default: + ptr = upb_WireReader_SkipValue(ptr, tag, stream); + if (!ptr) goto err; + } + } + +done: + if (type_id && msg) { + PyObject* field = PyObject_CallFunction( + s->unknown_field_type, "iiO", type_id, kUpb_WireType_Delimited, msg); + if (!field) goto err; + PyList_Append(self->fields, field); + Py_DECREF(field); + } + Py_XDECREF(msg); + return ptr; + +err: + Py_XDECREF(msg); + return NULL; +} + +static const char* PyUpb_UnknownFieldSet_BuildMessageSet( + PyUpb_UnknownFieldSet* self, upb_EpsCopyInputStream* stream, + const char* ptr) { + self->fields = PyList_New(0); + while (!upb_EpsCopyInputStream_IsDone(stream, &ptr)) { + uint32_t tag; + ptr = upb_WireReader_ReadTag(ptr, &tag); + if (!ptr) goto err; + if (tag == kUpb_MessageSet_StartItemTag) { + ptr = PyUpb_UnknownFieldSet_BuildMessageSetItem(self, stream, ptr); + } else { + ptr = upb_WireReader_SkipValue(ptr, tag, stream); + } + if (!ptr) goto err; + } + if (upb_EpsCopyInputStream_IsError(stream)) goto err; + return ptr; + +err: + Py_DECREF(self->fields); + self->fields = NULL; + return NULL; +} + +static const char* PyUpb_UnknownFieldSet_Build(PyUpb_UnknownFieldSet* self, + upb_EpsCopyInputStream* stream, + const char* ptr, + int group_number); + +static const char* PyUpb_UnknownFieldSet_BuildValue( + PyUpb_UnknownFieldSet* self, upb_EpsCopyInputStream* stream, + const char* ptr, int field_number, int wire_type, int group_number, + PyObject** data) { + switch (wire_type) { + case kUpb_WireType_Varint: { + uint64_t val; + ptr = upb_WireReader_ReadVarint(ptr, &val); + if (!ptr) return NULL; + *data = PyLong_FromUnsignedLongLong(val); + return ptr; + } + case kUpb_WireType_64Bit: { + uint64_t val; + ptr = upb_WireReader_ReadFixed64(ptr, &val); + *data = PyLong_FromUnsignedLongLong(val); + return ptr; + } + case kUpb_WireType_32Bit: { + uint32_t val; + ptr = upb_WireReader_ReadFixed32(ptr, &val); + *data = PyLong_FromUnsignedLongLong(val); + return ptr; + } + case kUpb_WireType_Delimited: { + int size; + ptr = upb_WireReader_ReadSize(ptr, &size); + if (!upb_EpsCopyInputStream_CheckDataSizeAvailable(stream, ptr, size)) { + return NULL; + } + const char* str = ptr; + ptr = upb_EpsCopyInputStream_ReadStringAliased(stream, &str, size); + *data = PyBytes_FromStringAndSize(str, size); + return ptr; + } + case kUpb_WireType_StartGroup: { + PyUpb_UnknownFieldSet* sub = PyUpb_UnknownFieldSet_NewBare(); + if (!sub) return NULL; + *data = &sub->ob_base; + return PyUpb_UnknownFieldSet_Build(sub, stream, ptr, field_number); + } + default: + assert(0); + *data = NULL; + return NULL; + } +} + +// For non-MessageSet we just build the unknown fields exactly as they exist on +// the wire. +static const char* PyUpb_UnknownFieldSet_Build(PyUpb_UnknownFieldSet* self, + upb_EpsCopyInputStream* stream, + const char* ptr, + int group_number) { + PyUpb_ModuleState* s = PyUpb_ModuleState_Get(); + self->fields = PyList_New(0); + while (!upb_EpsCopyInputStream_IsDone(stream, &ptr)) { + uint32_t tag; + ptr = upb_WireReader_ReadTag(ptr, &tag); + if (!ptr) goto err; + PyObject* data = NULL; + int field_number = upb_WireReader_GetFieldNumber(tag); + int wire_type = upb_WireReader_GetWireType(tag); + if (wire_type == kUpb_WireType_EndGroup) { + if (field_number != group_number) return NULL; + return ptr; + } + ptr = PyUpb_UnknownFieldSet_BuildValue(self, stream, ptr, field_number, + wire_type, group_number, &data); + if (!ptr) { + Py_XDECREF(data); + goto err; + } + assert(data); + PyObject* field = PyObject_CallFunction(s->unknown_field_type, "iiN", + field_number, wire_type, data); + PyList_Append(self->fields, field); + Py_DECREF(field); + } + if (upb_EpsCopyInputStream_IsError(stream)) goto err; + return ptr; + +err: + Py_DECREF(self->fields); + self->fields = NULL; + return NULL; +} + +static PyObject* PyUpb_UnknownFieldSet_New(PyTypeObject* type, PyObject* args, + PyObject* kwargs) { + char* kwlist[] = {"message", 0}; + PyObject* py_msg = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &py_msg)) { + return NULL; + } + + if (!PyUpb_Message_Verify(py_msg)) return NULL; + PyUpb_UnknownFieldSet* self = PyUpb_UnknownFieldSet_NewBare(); + upb_Message* msg = PyUpb_Message_GetIfReified(py_msg); + if (!msg) return &self->ob_base; + + size_t size; + const char* ptr = upb_Message_GetUnknown(msg, &size); + if (size == 0) return &self->ob_base; + + upb_EpsCopyInputStream stream; + upb_EpsCopyInputStream_Init(&stream, &ptr, size, true); + const upb_MessageDef* msgdef = PyUpb_Message_GetMsgdef(py_msg); + + bool ok; + if (upb_MessageDef_IsMessageSet(msgdef)) { + ok = PyUpb_UnknownFieldSet_BuildMessageSet(self, &stream, ptr) != NULL; + } else { + ok = PyUpb_UnknownFieldSet_Build(self, &stream, ptr, -1) != NULL; + } + + if (!ok) { + Py_DECREF(&self->ob_base); + return NULL; + } + + return &self->ob_base; +} + +static Py_ssize_t PyUpb_UnknownFieldSet_Length(PyObject* _self) { + PyUpb_UnknownFieldSet* self = (PyUpb_UnknownFieldSet*)_self; + return self->fields ? PyObject_Length(self->fields) : 0; +} + +static PyObject* PyUpb_UnknownFieldSet_GetItem(PyObject* _self, + Py_ssize_t index) { + PyUpb_UnknownFieldSet* self = (PyUpb_UnknownFieldSet*)_self; + if (!self->fields) { + PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index); + return NULL; + } + PyObject* ret = PyList_GetItem(self->fields, index); + if (ret) Py_INCREF(ret); + return ret; +} + +static PyType_Slot PyUpb_UnknownFieldSet_Slots[] = { + {Py_tp_new, &PyUpb_UnknownFieldSet_New}, + {Py_tp_dealloc, &PyUpb_UnknownFieldSet_Dealloc}, + {Py_sq_length, PyUpb_UnknownFieldSet_Length}, + {Py_sq_item, PyUpb_UnknownFieldSet_GetItem}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {0, NULL}, +}; + +static PyType_Spec PyUpb_UnknownFieldSet_Spec = { + PYUPB_MODULE_NAME ".UnknownFieldSet", // tp_name + sizeof(PyUpb_UnknownFieldSet), // tp_basicsize + 0, // tp_itemsize + Py_TPFLAGS_DEFAULT, // tp_flags + PyUpb_UnknownFieldSet_Slots, +}; + +// ----------------------------------------------------------------------------- +// Top Level +// ----------------------------------------------------------------------------- + +PyObject* PyUpb_UnknownFieldSet_CreateNamedTuple(void) { + PyObject* mod = NULL; + PyObject* namedtuple = NULL; + PyObject* ret = NULL; + + mod = PyImport_ImportModule("collections"); + if (!mod) goto done; + namedtuple = PyObject_GetAttrString(mod, "namedtuple"); + if (!namedtuple) goto done; + ret = PyObject_CallFunction(namedtuple, "s[sss]", "PyUnknownField", + "field_number", "wire_type", "data"); + +done: + Py_XDECREF(mod); + Py_XDECREF(namedtuple); + return ret; +} + +bool PyUpb_UnknownFields_Init(PyObject* m) { + PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m); + + s->unknown_fields_type = PyUpb_AddClass(m, &PyUpb_UnknownFieldSet_Spec); + s->unknown_field_type = PyUpb_UnknownFieldSet_CreateNamedTuple(); + + return s->unknown_fields_type && s->unknown_field_type; +}
diff --git a/python/unknown_fields.h b/python/unknown_fields.h new file mode 100644 index 0000000..85ea40c --- /dev/null +++ b/python/unknown_fields.h
@@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google LLC. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google LLC nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef PYUPB_UNKNOWN_FIELDS_H__ +#define PYUPB_UNKNOWN_FIELDS_H__ + +#include <stdbool.h> + +#include "python/python_api.h" + +PyObject* PyUpb_UnknownFields_New(PyObject* msg); + +bool PyUpb_UnknownFields_Init(PyObject* m); + +#endif // PYUPB_UNKNOWN_FIELDS_H__
diff --git a/python/version_script.lds b/python/version_script.lds new file mode 100644 index 0000000..7cb8300 --- /dev/null +++ b/python/version_script.lds
@@ -0,0 +1,6 @@ +message { + global: + PyInit__message; + local: + *; +};