Add bzlmod support for system_python.
#test-continuous
This creates a module extension for system python so that it can be used in bzlmod builds. The pip_parse setup (dynamically building a new repo) doesn't work in the new bzlmod model, so we use the regular pip extension pointed at system_python's interpreter.
Remaining work to get off WORKSPACE:
* The pip extension quickly fails in the case where there's no python available, unlike our system_python setup
* Our windows upb tests were never actually running python, and system_python doesn't work (and breaks under bzlmod due to the above issue)
* Some of the upb wheel tests still don't work on bzlmod yet because they depend on some very custom http_archive dependencies.
Closes #18750
Closes #23313
Closes #23307
PiperOrigin-RevId: 810995270
diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
index 40e49cf..420482b 100644
--- a/.github/workflows/test_python.yml
+++ b/.github/workflows/test_python.yml
@@ -29,8 +29,6 @@
matrix:
type: [ Pure, C++]
version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
- # TODO: Enable bzlmod once python headers are supported for python dist.
- bzlmod: [--noenable_bzlmod]
include:
- type: Pure
targets: //python/... //python:python_version_test
@@ -38,9 +36,12 @@
- type: C++
targets: //python/... //python:python_version_test
flags: --define=use_fast_cpp_protos=true
+ # Test using WORKSPACE with our oldest support Python version.
- version: "3.9"
+ nobzlmod: true
+ # Our 3.9 image has non-hermetic issues and can't be rebuilt. Use the old version.
+ image: us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.6.1-3.9-12e21b8dda91028bc14212a3ab582c7c4d149fac
- version: "3.10"
- continuous-only: true
- version: "3.11"
continuous-only: true
- version: "3.12"
@@ -51,10 +52,9 @@
targets: //python/... //python:aarch64_test
# TODO Enable this once conformance tests are fixed.
flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
- bzlmod: --noenable_bzlmod
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.6.1-aarch64-f0d1e209ed9369f69d93ce418990ecff3aa08d6f
- name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }}
+ name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }} ${{ matrix.nobzlmod && 'No Bzlmod' || '' }}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
@@ -66,10 +66,10 @@
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
uses: protocolbuffers/protobuf-ci/bazel-docker@v4
with:
- image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.6.1-{0}-12e21b8dda91028bc14212a3ab582c7c4d149fac', matrix.version) }}
+ image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.6.1-{0}-e1c2fe666ffa9b941b126541a26340d0ea787cea', matrix.version) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
- bazel: test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }} --test_env=KOKORO_PYTHON_VERSION
+ bazel: test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.nobzlmod && '--noenable_bzlmod' || '' }} --test_env=KOKORO_PYTHON_VERSION
macos:
@@ -78,8 +78,6 @@
matrix:
type: [ Pure, C++]
version: [ "3.12", "3.13" ]
- # TODO: Enable bzlmod once python headers are supported for python dist.
- bzlmod: [--noenable_bzlmod]
include:
- type: Pure
targets: //python/... //python:python_version_test
@@ -121,6 +119,6 @@
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }}
bazel: >-
- test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }}
+ test ${{ matrix.targets }} ${{ matrix.flags }}
--test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }}
--macos_minimum_os=11.0
diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml
index 52d942d..3f65de6 100644
--- a/.github/workflows/test_upb.yml
+++ b/.github/workflows/test_upb.yml
@@ -52,8 +52,7 @@
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '7.6.1' }}-12e21b8dda91028bc14212a3ab582c7c4d149fac
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: upb-bazel
- # TODO: Enable bzlmod once python headers are supported for python dist.
- bazel: test --noenable_bzlmod //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
+ bazel: test //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
exclude-targets: ${{ matrix.config.exclude-targets }}
linux-gcc:
@@ -73,7 +72,7 @@
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-gcc"
bazel: >-
- test --noenable_bzlmod -c opt
+ test -c opt
--copt="-Wno-error=maybe-uninitialized" --java_runtime_version=remotejdk_11
//bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
@@ -92,7 +91,7 @@
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-windows"
- bazel: test --noenable_bzlmod //upb/... //upb_generator/... //python/...
+ bazel: test //upb/... //upb_generator/... //python/... --noenable_bzlmod
version: 7.6.1
exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
@@ -121,7 +120,7 @@
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-macos"
- bazel: ${{ matrix.config.bazel-command }} --noenable_bzlmod ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
+ bazel: ${{ matrix.config.bazel-command }} ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
version: 7.6.1
no-python:
diff --git a/MODULE.bazel b/MODULE.bazel
index 322a304..827c35a 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -123,29 +123,25 @@
"3.10",
"3.11",
"3.12",
+ "3.13",
]
-# TODO: Support hermetic / system python in bzlmod.
-python = use_extension("@rules_python//python/extensions:python.bzl", "python")
-
-[
- python.toolchain(
- python_version = python_version,
- )
- for python_version in SUPPORTED_PYTHON_VERSIONS
-]
-
-python.defaults(python_version = SUPPORTED_PYTHON_VERSIONS[-1])
-use_repo(
- python,
- system_python = "python_{}".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")),
+# TODO: Replace system_python with hermetic_python.
+# TODO: Remove dev_dependency once system_python is no longer used and we support
+# python/upb in Bazel.
+system_python = use_extension("//python/dist:system_python.bzl", "system_python_extension", dev_dependency = True)
+system_python.find(
+ name = "system_python",
+ minimum = "3.9",
)
+use_repo(system_python, "system_python")
pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True)
[
pip.parse(
hub_name = "protobuf_pip_deps",
+ python_interpreter_target = "@system_python//:interpreter",
python_version = python_version,
requirements_lock = "//python:requirements.txt",
)
diff --git a/cmake/dependencies_generator.py b/cmake/dependencies_generator.py
index 875a7f0..22a9e53 100644
--- a/cmake/dependencies_generator.py
+++ b/cmake/dependencies_generator.py
@@ -46,6 +46,9 @@
def bundle_fetch(self, *args, **kwargs):
pass
+ def find(self, *args, **kwargs):
+ pass
+
def empty_func(*args, **kwargs):
pass
diff --git a/python/dist/system_python.bzl b/python/dist/system_python.bzl
index 5f0fdf5..2a1c564 100644
--- a/python/dist/system_python.bzl
+++ b/python/dist/system_python.bzl
@@ -69,8 +69,10 @@
"""
_build_file = """
+load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
load("@bazel_skylib//lib:selects.bzl", "selects")
load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+load("@rules_python//python:py_runtime.bzl", "py_runtime")
load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair")
cc_library(
@@ -269,3 +271,23 @@
"minimum_python_version": attr.string(default = "3.9"),
},
)
+
+def _system_python_extension(ctx):
+ for mod in ctx.modules:
+ for py in mod.tags.find:
+ system_python(
+ name = py.name,
+ minimum_python_version = py.minimum,
+ )
+
+find = tag_class(attrs = {
+ "name": attr.string(doc = "Supported versions of python to find"),
+ "minimum": attr.string(),
+})
+
+system_python_extension = module_extension(
+ implementation = _system_python_extension,
+ tag_classes = {
+ "find": find,
+ },
+)
