Create targets for UPB release
PiperOrigin-RevId: 441496547
diff --git a/WORKSPACE b/WORKSPACE
index 0bd8479..823037b 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1,8 +1,9 @@
workspace(name = "upb")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("//bazel:python_downloads.bzl", "python_source_archive", "python_nuget_package")
load("//bazel:workspace_deps.bzl", "upb_deps")
-load("//bazel:workspace_defs.bzl", "system_python")
+load("//bazel:system_python.bzl", "system_python")
upb_deps()
@@ -62,3 +63,42 @@
load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
rules_fuzzing_init()
+
+#Python Downloads
+
+python_source_archive(
+ name = "python-3.7.0",
+ sha256 = "85bb9feb6863e04fb1700b018d9d42d1caac178559ffa453d7e6a436e259fd0d",
+)
+python_nuget_package(
+ name = "nuget_python_i686_3.7.0",
+ sha256 = "a8bb49fa1ca62ad55430fcafaca1b58015e22943e66b1a87d5e7cef2556c6a54",
+)
+python_nuget_package(
+ name = "nuget_python_x86-64_3.7.0",
+ sha256 = "66eb796a5bdb1e6787b8f655a1237a6b6964af2115b7627cf4f0032cf068b4b2",
+)
+python_nuget_package(
+ name = "nuget_python_i686_3.8.0",
+ sha256 = "87a6481f5eef30b42ac12c93f06f73bd0b8692f26313b76a6615d1641c4e7bca",
+)
+python_nuget_package(
+ name = "nuget_python_x86-64_3.8.0",
+ sha256 = "96c61321ce90dd053c8a04f305a5f6cc6d91350b862db34440e4a4f069b708a0",
+)
+python_nuget_package(
+ name = "nuget_python_i686_3.9.0",
+ sha256 = "229abecbe49dc08fe5709e0b31e70edfb3b88f23335ebfc2904c44f940fd59b6",
+)
+python_nuget_package(
+ name = "nuget_python_x86-64_3.9.0",
+ sha256 = "6af58a733e7dfbfcdd50d55788134393d6ffe7ab8270effbf724bdb786558832",
+)
+python_nuget_package(
+ name = "nuget_python_i686_3.10.0",
+ sha256 = "e115e102eb90ce160ab0ef7506b750a8d7ecc385bde0a496f02a54337a8bc333",
+)
+python_nuget_package(
+ name = "nuget_python_x86-64_3.10.0",
+ sha256 = "4474c83c25625d93e772e926f95f4cd398a0abbb52793625fa30f39af3d2cc00",
+)
diff --git a/bazel/py_extension.bzl b/bazel/py_extension.bzl
deleted file mode 100644
index eacd39b..0000000
--- a/bazel/py_extension.bzl
+++ /dev/null
@@ -1,38 +0,0 @@
-load(
- "//bazel:build_defs.bzl",
- "UPB_DEFAULT_COPTS",
-)
-
-def py_extension(name, srcs, deps = []):
- version_script = name + "_version_script.lds"
- symbol = "PyInit_" + name
- native.genrule(
- name = "gen_" + version_script,
- outs = [version_script],
- cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@",
- )
-
- native.cc_binary(
- name = name,
- srcs = srcs,
- copts = UPB_DEFAULT_COPTS + [
- # The Python API requires patterns that are ISO C incompatible, like
- # casts between function pointers and object pointers.
- "-Wno-pedantic",
- ],
- # We use a linker script to hide all symbols except the entry point for
- # the module.
- linkopts = select({
- "@platforms//os:linux": ["-Wl,--version-script,$(location :" + version_script + ")"],
- "@platforms//os:macos": [
- "-Wl,-exported_symbol",
- "-Wl,_" + symbol,
- ],
- }),
- linkshared = True,
- linkstatic = True,
- deps = deps + [
- ":" + version_script,
- "@system_python//:python_headers",
- ],
- )
diff --git a/bazel/pyproto_test_wrapper.bzl b/bazel/pyproto_test_wrapper.bzl
index 629ee9b..3092314 100644
--- a/bazel/pyproto_test_wrapper.bzl
+++ b/bazel/pyproto_test_wrapper.bzl
@@ -9,7 +9,8 @@
main = src,
data = ["@com_google_protobuf//:testdata"],
deps = [
- "//python:message_ext",
+ "//python:_api_implementation",
+ "//python:_message",
"@com_google_protobuf//:python_common_test_protos",
"@com_google_protobuf//:python_specific_test_protos",
"@com_google_protobuf//:python_srcs",
diff --git a/bazel/python_downloads.bzl b/bazel/python_downloads.bzl
new file mode 100644
index 0000000..e237c93
--- /dev/null
+++ b/bazel/python_downloads.bzl
@@ -0,0 +1,84 @@
+"""Helper methods to download different python versions"""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+limited_api_build_file = """
+cc_library(
+ name = "python_headers",
+ hdrs = glob(["**/Include/**/*.h"]),
+ strip_include_prefix = "Python-{}/Include",
+ visibility = ["//visibility:public"],
+)
+"""
+
+def python_source_archive(name, sha256):
+ """Helper method to create a python_headers target that will work for linux and macos.
+
+ Args:
+ name: The name of the target, should be in the form python_{VERSION}
+ sha256: The sha256 of the python package for the specified version
+ """
+ version = name.split("-")[1]
+ http_archive(
+ name = name,
+ urls = [
+ "https://www.python.org/ftp/python/{0}/Python-{0}.tgz"
+ .format(version),
+ ],
+ sha256 = sha256,
+ build_file_content = limited_api_build_file.format(version),
+ patch_cmds = [
+ "echo '#define SIZEOF_WCHAR_T 4' > Python-{}/Include/pyconfig.h"
+ .format(version),
+ ],
+ )
+
+nuget_build_file = """
+cc_import(
+ name = "python_full_api",
+ hdrs = glob(["**/*.h"]),
+ shared_library = "python{0}.dll",
+ interface_library = "libs/python{0}.lib",
+ visibility = ["@upb//python:__pkg__"],
+)
+
+cc_import(
+ name = "python_limited_api",
+ hdrs = glob(["**/*.h"]),
+ shared_library = "python{1}.dll",
+ interface_library = "libs/python{1}.lib",
+ visibility = ["@upb//python:__pkg__"],
+)
+"""
+
+def python_nuget_package(name, sha256):
+ """Helper method to create full and limited api dependencies for windows using nuget
+
+ Args:
+ name: The name of the target, should be in the form nuget_python_{CPU}_{VERSION}
+ sha256: The sha256 of the nuget package for that version
+ """
+ cpu = name.split("_")[2]
+ version = name.split("_")[3]
+
+ full_api_lib_number = version.split(".")[0] + version.split(".")[1]
+ limited_api_lib_number = version.split(".")[0]
+
+ folder_name_dict = {
+ "i686": "pythonx86",
+ "x86-64": "python",
+ }
+
+ http_archive(
+ name = name,
+ urls = [
+ "https://www.nuget.org/api/v2/package/{}/{}"
+ .format(folder_name_dict[cpu], version),
+ ],
+ sha256 = sha256,
+ strip_prefix = "tools",
+ build_file_content =
+ nuget_build_file.format(full_api_lib_number, limited_api_lib_number),
+ type = "zip",
+ patch_cmds = ["cp -r include/* ."],
+ )
diff --git a/bazel/workspace_defs.bzl b/bazel/system_python.bzl
similarity index 86%
rename from bazel/workspace_defs.bzl
rename to bazel/system_python.bzl
index 03655b2..9e20b1f 100644
--- a/bazel/workspace_defs.bzl
+++ b/bazel/system_python.bzl
@@ -27,6 +27,7 @@
_build_file = """
load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+
cc_library(
name = "python_headers",
hdrs = glob(["python/**/*.h"]),
@@ -36,7 +37,7 @@
py_runtime(
name = "py3_runtime",
- interpreter_path = "%s",
+ interpreter_path = "{}",
python_version = "PY3",
)
@@ -52,6 +53,11 @@
)
"""
+def _get_python_version(repository_ctx):
+ py_program = "import sys; print(str(sys.version_info.major) + str(sys.version_info.minor))"
+ result = repository_ctx.execute(["python3", "-c", py_program])
+ return (result.stdout).strip()
+
def _get_config_var(repository_ctx, name):
py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')"
result = repository_ctx.execute(["python3", "-c", py_program % (name)])
@@ -63,7 +69,9 @@
path = _get_config_var(repository_ctx, "INCLUDEPY")
repository_ctx.symlink(path, "python")
python3 = repository_ctx.which("python3")
- repository_ctx.file("BUILD.bazel", _build_file % python3)
+ python_version = _get_python_version(repository_ctx)
+ repository_ctx.file("BUILD.bazel", _build_file.format(python3))
+ repository_ctx.file("version.bzl", "SYSTEM_PYTHON_VERSION = {}".format(python_version))
# The system_python() repository rule exposes Python headers from the system.
#
diff --git a/bazel/workspace_deps.bzl b/bazel/workspace_deps.bzl
index db3afd7..ca9b756 100644
--- a/bazel/workspace_deps.bzl
+++ b/bazel/workspace_deps.bzl
@@ -14,7 +14,7 @@
maybe(
git_repository,
name = "com_google_protobuf",
- commit = "2f91da585e96a7efe43505f714f03c7716a94ecb",
+ commit = "a69354f31b253856689ae765a9ea3217ec001873",
remote = "https://github.com/protocolbuffers/protobuf.git",
patches = [
"//bazel:protobuf.patch",
diff --git a/cmake/make_cmakelists.py b/cmake/make_cmakelists.py
index 29c73bb..655d809 100755
--- a/cmake/make_cmakelists.py
+++ b/cmake/make_cmakelists.py
@@ -234,6 +234,12 @@
def register_toolchains(self, toolchain):
pass
+ def python_source_archive(self, **kwargs):
+ pass
+
+ def python_nuget_package(self, **kwargs):
+ pass
+
class Converter(object):
def __init__(self):
diff --git a/python/BUILD b/python/BUILD
index ac97113..ac96b6c 100644
--- a/python/BUILD
+++ b/python/BUILD
@@ -24,13 +24,154 @@
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# begin:github_only
-load("//bazel:py_proto_library.bzl", "py_proto_library")
-load("//bazel:py_extension.bzl", "py_extension")
-load("@rules_python//python:packaging.bzl", "py_wheel")
+load("//python:py_extension.bzl", "py_extension")
# end:github_only
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
+load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS")
+
licenses(["notice"])
+package(
+ default_visibility = ["//python/dist:__pkg__"],
+)
+
+LIMITED_API_FLAG_SELECT = {
+ ":limited_api_3.7": ["-DPy_LIMITED_API=0x03070000"],
+ ":limited_api_3.10": ["-DPy_LIMITED_API=0x030a0000"],
+ "//conditions:default": [],
+}
+
+bool_flag(
+ name = "limited_api",
+ build_setting_default = False,
+)
+
+string_flag(
+ name = "python_version",
+ build_setting_default = "system",
+ values = [
+ "system",
+ "37",
+ "38",
+ "39",
+ "310",
+ ],
+)
+
+config_setting(
+ name = "limited_api_3.7",
+ flag_values = {
+ ":limited_api": "True",
+ ":python_version": "37",
+ },
+)
+
+config_setting(
+ name = "full_api_3.7_win32",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "37",
+ },
+ values = {"cpu": "win32"},
+)
+
+config_setting(
+ name = "full_api_3.7_win64",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "37",
+ },
+ values = {"cpu": "win64"},
+)
+
+selects.config_setting_group(
+ name = "full_api_3.7",
+ match_any = [
+ ":full_api_3.7_win32",
+ ":full_api_3.7_win64",
+ ],
+)
+
+config_setting(
+ name = "full_api_3.8_win32",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "38",
+ },
+ values = {"cpu": "win32"},
+)
+
+config_setting(
+ name = "full_api_3.8_win64",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "38",
+ },
+ values = {"cpu": "win64"},
+)
+
+selects.config_setting_group(
+ name = "full_api_3.8",
+ match_any = [
+ ":full_api_3.8_win32",
+ ":full_api_3.8_win64",
+ ],
+)
+
+config_setting(
+ name = "full_api_3.9_win32",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "39",
+ },
+ values = {"cpu": "win32"},
+)
+
+config_setting(
+ name = "full_api_3.9_win64",
+ flag_values = {
+ ":limited_api": "False",
+ ":python_version": "39",
+ },
+ values = {"cpu": "win64"},
+)
+
+selects.config_setting_group(
+ name = "full_api_3.9",
+ match_any = [
+ "full_api_3.9_win32",
+ ":full_api_3.9_win64",
+ ],
+)
+
+config_setting(
+ name = "limited_api_3.10_win32",
+ flag_values = {
+ ":limited_api": "True",
+ ":python_version": "310",
+ },
+ values = {"cpu": "win32"},
+)
+
+config_setting(
+ name = "limited_api_3.10_win64",
+ flag_values = {
+ ":limited_api": "True",
+ ":python_version": "310",
+ },
+ values = {"cpu": "win64"},
+)
+
+selects.config_setting_group(
+ name = "limited_api_3.10",
+ match_any = [
+ ":limited_api_3.10_win32",
+ ":limited_api_3.10_win64",
+ ],
+)
+
py_extension(
name = "_message",
srcs = [
@@ -50,10 +191,15 @@
"message.h",
"protobuf.c",
"protobuf.h",
- "python.h",
+ "python_api.h",
"repeated.c",
"repeated.h",
],
+ copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [
+ # The Python API requires patterns that are ISO C incompatible, like
+ # casts between function pointers and object pointers.
+ "-Wno-pedantic",
+ ],
deps = [
"//:descriptor_upb_proto_reflection",
"//:reflection",
@@ -68,97 +214,13 @@
py_extension(
name = "_api_implementation",
- srcs = ["api_implementation.c"],
-)
-
-# begin:github_only
-
-py_test(
- name = "minimal_test",
srcs = [
- "minimal_test.py",
+ "api_implementation.c",
+ "python_api.h",
],
- imports = ["."],
- legacy_create_init = False,
- deps = [
- "//python:message_ext",
- "@com_google_protobuf//:python_common_test_protos",
- "@com_google_protobuf//:python_specific_test_protos",
- "@com_google_protobuf//:python_srcs",
+ copts = UPB_DEFAULT_COPTS + select(LIMITED_API_FLAG_SELECT) + [
+ # The Python API requires patterns that are ISO C incompatible, like
+ # casts between function pointers and object pointers.
+ "-Wno-pedantic",
],
)
-
-# Copy the extensions into the location recognized by Python.
-# .abi3.so indicates use of the limited API, and cross-version ABI compatibility.
-EXT_SUFFIX = ".abi3.so"
-
-genrule(
- name = "copy_message",
- srcs = [":_message"],
- outs = ["google/protobuf/pyext/_message" + EXT_SUFFIX],
- cmd = "cp $< $@",
-)
-
-genrule(
- name = "copy_api_implementation",
- srcs = [":_api_implementation"],
- outs = ["google/protobuf/internal/_api_implementation" + EXT_SUFFIX],
- cmd = "cp $< $@",
- visibility = ["//python:__subpackages__"],
-)
-
-filegroup(
- name = "extension_files",
- srcs = [
- "google/protobuf/pyext/_message" + EXT_SUFFIX,
- "google/protobuf/internal/_api_implementation" + EXT_SUFFIX,
- ],
-)
-
-py_library(
- name = "message_ext",
- data = [":extension_files"],
- imports = ["."],
- visibility = ["//python:__subpackages__"],
-)
-
-py_proto_library(
- name = "well_known_proto_pb2",
- deps = [
- "@com_google_protobuf//:any_proto",
- "@com_google_protobuf//:api_proto",
- "@com_google_protobuf//:compiler_plugin_proto",
- "@com_google_protobuf//:descriptor_proto",
- "@com_google_protobuf//:duration_proto",
- "@com_google_protobuf//:empty_proto",
- "@com_google_protobuf//:field_mask_proto",
- "@com_google_protobuf//:source_context_proto",
- "@com_google_protobuf//:struct_proto",
- "@com_google_protobuf//:timestamp_proto",
- "@com_google_protobuf//:type_proto",
- "@com_google_protobuf//:wrappers_proto",
- ],
-)
-
-py_wheel(
- name = "binary_wheel",
- abi = "abi3",
- distribution = "protobuf",
- # TODO(https://github.com/protocolbuffers/upb/issues/502): we need to make
- # this a select() that is calculated from the platform we are actually
- # building on.
- platform = "manylinux2014_x86_64",
- python_tag = "cp36",
- strip_path_prefixes = ["python/"],
- version = "4.20.0",
- deps = [
- ":extension_files",
- ":well_known_proto_pb2",
- # TODO(https://github.com/protocolbuffers/upb/issues/503): currently
- # this includes the unit tests. We should filter these out so we are
- # only distributing true source files.
- "@com_google_protobuf//:python_srcs",
- ],
-)
-
-# end:github_only
diff --git a/python/api_implementation.c b/python/api_implementation.c
index 9f96b93..744f2be 100644
--- a/python/api_implementation.c
+++ b/python/api_implementation.c
@@ -25,7 +25,7 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-#include <Python.h>
+#include "python/python_api.h"
static struct PyModuleDef module_def = {
PyModuleDef_HEAD_INIT,
diff --git a/python/descriptor.h b/python/descriptor.h
index 6cb6c1c..581e1c7 100644
--- a/python/descriptor.h
+++ b/python/descriptor.h
@@ -30,7 +30,7 @@
#include <stdbool.h>
-#include "python/python.h"
+#include "python/python_api.h"
#include "upb/def.h"
typedef enum {
diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel
new file mode 100644
index 0000000..33e6287
--- /dev/null
+++ b/python/dist/BUILD.bazel
@@ -0,0 +1,152 @@
+# Copyright (c) 2009-2022, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of Google LLC nor the
+# names of its contributors may be used to endorse or promote products
+# derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load("//bazel:py_proto_library.bzl", "py_proto_library")
+load(":dist.bzl", "py_dist", "py_dist_module")
+load("@bazel_skylib//lib:selects.bzl", "selects")
+load("@com_google_protobuf//:protobuf_version.bzl", "PROTOBUF_VERSION")
+load("@rules_python//python:packaging.bzl", "py_wheel")
+
+licenses(["notice"])
+
+py_dist_module(
+ name = "message_mod",
+ extension = "//python:_message_binary",
+ module_name = "google.protobuf.pyext._message",
+)
+
+py_dist_module(
+ name = "api_implementation_mod",
+ extension = "//python:_api_implementation_binary",
+ module_name = "google.protobuf.internal.api_implementation",
+)
+
+py_proto_library(
+ name = "well_known_proto_py_pb2",
+ deps = [
+ "@com_google_protobuf//:any_proto",
+ "@com_google_protobuf//:api_proto",
+ "@com_google_protobuf//:compiler_plugin_proto",
+ "@com_google_protobuf//:descriptor_proto",
+ "@com_google_protobuf//:duration_proto",
+ "@com_google_protobuf//:empty_proto",
+ "@com_google_protobuf//:field_mask_proto",
+ "@com_google_protobuf//:source_context_proto",
+ "@com_google_protobuf//:struct_proto",
+ "@com_google_protobuf//:timestamp_proto",
+ "@com_google_protobuf//:type_proto",
+ "@com_google_protobuf//:wrappers_proto",
+ ],
+)
+
+config_setting(
+ name = "aarch64_cpu",
+ values = {"cpu": "linux-aarch_64"},
+)
+
+config_setting(
+ name = "x86_64_cpu",
+ values = {"cpu": "linux-x86_64"},
+)
+
+config_setting(
+ name = "osx-x86_64_cpu",
+ values = {"cpu": "osx-x86_64"},
+)
+
+config_setting(
+ name = "win32_cpu",
+ values = {"cpu": "win32"},
+)
+
+config_setting(
+ name = "win64_cpu",
+ values = {"cpu": "win64"},
+)
+
+py_wheel(
+ name = "binary_wheel",
+ abi = select({
+ "//python:full_api_3.7": "cp37m",
+ "//python:full_api_3.8": "cp38",
+ "//python:full_api_3.9": "cp39",
+ "//conditions:default": "abi3",
+ }),
+ distribution = "protobuf",
+ platform = select({
+ ":x86_64_cpu": "manylinux2014_x86_64",
+ ":aarch64_cpu": "manylinux2014_aarch64",
+ ":osx-x86_64_cpu": "macosx_10_9_universal",
+ ":win32_cpu": "win32",
+ ":win64_cpu": "win_amd64",
+ "//conditions:default": "any",
+ }),
+ python_tag = selects.with_or({
+ ("//python:limited_api_3.7", "//python:full_api_3.7"): "cp37",
+ "//python:full_api_3.8": "cp38",
+ "//python:full_api_3.9": "cp39",
+ "//python:limited_api_3.10": "cp310",
+ "//conditions:default": "system",
+ }),
+ strip_path_prefixes = ["python/"],
+ version = PROTOBUF_VERSION,
+ deps = [
+ ":message_mod",
+ ":api_implementation_mod",
+ ":well_known_proto_py_pb2",
+ #TODO(https://github.com/protocolbuffers/upb/issues/503): currently
+ # this includes the unit tests. We should filter these out so we are
+ # only distributing true source files.
+ "@com_google_protobuf//:python_srcs",
+ ],
+)
+
+py_dist(
+ name = "dist",
+ binary_wheel = ":binary_wheel",
+ full_api_cpus = [
+ "win32",
+ "win64",
+ ],
+ # Windows needs version-specific wheels until 3.10.
+ full_api_versions = [
+ "37",
+ "38",
+ "39",
+ ],
+ # Limited API: these wheels will satisfy any Python version >= the
+ # given version.
+ #
+ # Technically the limited API doesn't have the functions we need until
+ # 3.10, but on Linux we can get away with using 3.7 (see ../python_api.h for
+ # details).
+ limited_api_wheels = {
+ "win32": "310",
+ "win64": "310",
+ "linux-x86_64": "37",
+ "linux-aarch_64": "37",
+ },
+ tags = ["manual"],
+)
diff --git a/python/dist/dist.bzl b/python/dist/dist.bzl
new file mode 100644
index 0000000..6760a38
--- /dev/null
+++ b/python/dist/dist.bzl
@@ -0,0 +1,145 @@
+"""Rules to create python distribution files and properly name them"""
+
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
+load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION")
+
+def _get_suffix(limited_api, python_version, cpu):
+ suffix = "pyd" if ("win" in cpu) else "so"
+
+ if limited_api == True:
+ if "win" not in cpu:
+ suffix = "abi3." + suffix
+ return "." + suffix
+
+ if "win32" in cpu or "win64" in cpu:
+ if "win32" in cpu:
+ abi = "win32"
+ elif "win64" in cpu:
+ abi = "win_amd64"
+ else:
+ fail("Unsupported CPU: " + cpu)
+ return ".cp{}-{}.{}".format(python_version, abi, suffix)
+
+ if python_version == "system":
+ python_version = SYSTEM_PYTHON_VERSION
+ if int(python_version) < 38:
+ python_version += "m"
+ abis = {
+ "darwin": "darwin",
+ "osx-x86_64": "darwin",
+ "osx-aarch_64": "darwin",
+ "linux-aarch_64": "aarch64-linux-gnu",
+ "linux-x86_64": "x86_64-linux-gnu",
+ "k8": "x86_64-linux-gnu",
+ }
+
+ return ".cpython-{}-{}.{}".format(python_version, abis[cpu], suffix)
+
+ fail("Unsupported combination of flags")
+
+def _py_dist_module_impl(ctx):
+ base_filename = ctx.attr.module_name.replace(".", "/")
+ suffix = _get_suffix(
+ limited_api = ctx.attr._limited_api[BuildSettingInfo].value,
+ python_version = ctx.attr._python_version[BuildSettingInfo].value,
+ cpu = ctx.var["TARGET_CPU"],
+ )
+ filename = base_filename + suffix
+ file = ctx.actions.declare_file(filename)
+ src = ctx.attr.extension[DefaultInfo].files.to_list()[0]
+ ctx.actions.run(
+ executable = "cp",
+ arguments = [src.path, file.path],
+ inputs = [src],
+ outputs = [file],
+ )
+ return [
+ DefaultInfo(files = depset([file])),
+ ]
+
+_py_dist_module_rule = rule(
+ output_to_genfiles = True,
+ implementation = _py_dist_module_impl,
+ fragments = ["cpp"],
+ attrs = {
+ "module_name": attr.string(mandatory = True),
+ "extension": attr.label(
+ mandatory = True,
+ providers = [CcInfo],
+ ),
+ "_limited_api": attr.label(default = "//python:limited_api"),
+ "_python_version": attr.label(default = "//python:python_version"),
+ "_cc_toolchain": attr.label(
+ default = "@bazel_tools//tools/cpp:current_cc_toolchain",
+ ),
+ },
+)
+
+def py_dist_module(name, module_name, extension):
+ file_rule = name + "_file"
+ _py_dist_module_rule(
+ name = file_rule,
+ module_name = module_name,
+ extension = extension,
+ )
+
+ # TODO(haberman): needed?
+ native.py_library(
+ name = name,
+ data = [":" + file_rule],
+ imports = ["."],
+ )
+
+def _py_dist_transition_impl(settings, attr):
+ _ignore = (settings) # @unused
+ transitions = []
+
+ for cpu, version in attr.limited_api_wheels.items():
+ transitions.append({
+ "//command_line_option:cpu": cpu,
+ "//python:python_version": version,
+ "//python:limited_api": True,
+ })
+
+ for version in attr.full_api_versions:
+ for cpu in attr.full_api_cpus:
+ transitions.append({
+ "//command_line_option:cpu": cpu,
+ "//python:python_version": version,
+ "//python:limited_api": False,
+ })
+
+ return transitions
+
+_py_dist_transition = transition(
+ implementation = _py_dist_transition_impl,
+ inputs = [],
+ outputs = [
+ "//command_line_option:cpu",
+ "//python:python_version",
+ "//python:limited_api",
+ ],
+)
+
+def _py_dist_impl(ctx):
+ return [
+ DefaultInfo(files = depset(
+ transitive = [dep[DefaultInfo].files for dep in ctx.attr.binary_wheel],
+ )),
+ ]
+
+py_dist = rule(
+ implementation = _py_dist_impl,
+ attrs = {
+ "binary_wheel": attr.label(
+ mandatory = True,
+ cfg = _py_dist_transition,
+ ),
+ "limited_api_wheels": attr.string_dict(),
+ "full_api_versions": attr.string_list(),
+ "full_api_cpus": attr.string_list(),
+ "_allowlist_function_transition": attr.label(
+ default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+ ),
+ },
+)
diff --git a/python/extension_dict.h b/python/extension_dict.h
index f8a8fc4..a21822c 100644
--- a/python/extension_dict.h
+++ b/python/extension_dict.h
@@ -30,7 +30,7 @@
#include <stdbool.h>
-#include "python/python.h"
+#include "python/python_api.h"
PyObject* PyUpb_ExtensionDict_New(PyObject* msg);
diff --git a/python/map.h b/python/map.h
index 0c03ed0..aaa4e20 100644
--- a/python/map.h
+++ b/python/map.h
@@ -30,7 +30,7 @@
#include <stdbool.h>
-#include "python/python.h"
+#include "python/python_api.h"
#include "upb/def.h"
// Creates a new repeated field stub for field `f` of message object `parent`.
diff --git a/python/protobuf.h b/python/protobuf.h
index 0b87646..262f33e 100644
--- a/python/protobuf.h
+++ b/python/protobuf.h
@@ -31,7 +31,7 @@
#include <stdbool.h>
#include "python/descriptor.h"
-#include "python/python.h"
+#include "python/python_api.h"
#include "upb/table_internal.h"
diff --git a/python/py_extension.bzl b/python/py_extension.bzl
new file mode 100644
index 0000000..2fea049
--- /dev/null
+++ b/python/py_extension.bzl
@@ -0,0 +1,70 @@
+"""Macro to support py_extension
+"""
+
+def py_extension(name, srcs, copts, deps = []):
+ """Creates a C++ library to extend python
+
+ Args:
+ name: Name of the target
+ srcs: List of source files to create the target
+ copts: List of C++ compile options to use
+ deps: Libraries that the target depends on
+ """
+
+ version_script = name + "_version_script.lds"
+ symbol = "PyInit_" + name
+ native.genrule(
+ name = "gen_" + version_script,
+ outs = [version_script],
+ cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@",
+ )
+
+ native.cc_binary(
+ name = name + "_binary",
+ srcs = srcs,
+ copts = copts,
+ linkopts = select({
+ "//python/dist:osx-x86_64_cpu": ["-undefined", "dynamic_lookup"],
+ "//conditions:default": [],
+ }),
+ linkshared = True,
+ linkstatic = True,
+ deps = deps + [
+ ":" + version_script,
+ ] + select({
+ "//python:limited_api_3.7": ["@python-3.7.0//:python_headers"],
+ "//python:full_api_3.7_win32": ["@nuget_python_i686_3.7.0//:python_full_api"],
+ "//python:full_api_3.7_win64": ["@nuget_python_x86-64_3.7.0//:python_full_api"],
+ "//python:full_api_3.8_win32": ["@nuget_python_i686_3.8.0//:python_full_api"],
+ "//python:full_api_3.8_win64": ["@nuget_python_x86-64_3.8.0//:python_full_api"],
+ "//python:full_api_3.9_win32": ["@nuget_python_i686_3.9.0//:python_full_api"],
+ "//python:full_api_3.9_win64": ["@nuget_python_x86-64_3.9.0//:python_full_api"],
+ "//python:limited_api_3.10_win32": ["@nuget_python_i686_3.10.0//:python_limited_api"],
+ "//python:limited_api_3.10_win64": ["@nuget_python_x86-64_3.10.0//:python_limited_api"],
+ "//conditions:default": ["@system_python//:python_headers"],
+ }),
+ )
+
+ EXT_SUFFIX = ".abi3.so"
+
+ module_name_map = {
+ "_message": "pyext",
+ "_api_implementation": "internal",
+ }
+
+ output_file = "google/protobuf/" + module_name_map[name] + "/" + name + EXT_SUFFIX
+
+ native.genrule(
+ name = "copy" + name,
+ srcs = [":" + name + "_binary"],
+ outs = [output_file],
+ cmd = "cp $< $@",
+ visibility = ["//python:__subpackages__"],
+ )
+
+ native.py_library(
+ name = name,
+ data = [output_file],
+ imports = ["."],
+ visibility = ["//python:__subpackages__"],
+ )
diff --git a/python/python.h b/python/python_api.h
similarity index 61%
rename from python/python.h
rename to python/python_api.h
index 7eef3e5..e8a8fcf 100644
--- a/python/python.h
+++ b/python/python_api.h
@@ -28,15 +28,33 @@
#ifndef PYUPB_PYTHON_H__
#define PYUPB_PYTHON_H__
-// We restrict ourselves to the limited API, so that we will be ABI-compatible
-// with any version of Python >= 3.6.1 (3.6.1 introduce PySlice_Unpack())
-#define Py_LIMITED_API 0x03060100
-#include <Python.h>
+// We restrict ourselves to the limited API, so that a single build can be
+// ABI-compatible with a wide range of Python versions.
+//
+// The build system will define Py_LIMITED_API as appropriate (see BUILD). We
+// only want to define it for our distribution packages, since we can do some
+// extra assertions when Py_LIMITED_API is not defined. Also Py_LIMITED_API is
+// incompatible with Py_DEBUG.
-// This function was not officially added to the limited API until Python 3.10.
-// But in practice it has been stable since Python 3.1. See:
+// #define Py_LIMITED_API <val> // Defined by build system when appropriate.
+
+#include "Python.h"
+
+// Ideally we could restrict ourselves to the limited API of 3.7, but this is
+// a very important function that was not officially added to the limited API
+// until 3.10. Without this function, there is no way of getting data from a
+// Python `str` object without a copy.
+//
+// While this function was not *officially* added to the limited API until
+// Python 3.10, In practice it has been stable since Python 3.1.
// https://bugs.python.org/issue41784
+//
+// On Linux, ELF lets us get away with using this function with the limited
+// API prior to 3.10.
+
+#if defined(__linux__) && defined(Py_LIMITED_API) && Py_LIMITED_API < 0x03100000
PyAPI_FUNC(const char*)
PyUnicode_AsUTF8AndSize(PyObject* unicode, Py_ssize_t* size);
+#endif
#endif // PYUPB_PYTHON_H__
diff --git a/python/repeated.h b/python/repeated.h
index 9cde3ea..5d74bd2 100644
--- a/python/repeated.h
+++ b/python/repeated.h
@@ -30,7 +30,7 @@
#include <stdbool.h>
-#include "python/python.h"
+#include "python/python_api.h"
#include "upb/def.h"
// Creates a new repeated field stub for field `f` of message object `parent`.
