Merge pull request #8479 from jtattermusch/run_python_aarch64_tests
Run python aarch64 tests on the CI
diff --git a/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh
index 806c90f..115478b 100755
--- a/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh
+++ b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh
@@ -5,11 +5,18 @@
# go to the repo root
cd $(dirname $0)/../../../..
+if [[ -t 0 ]]; then
+ DOCKER_TTY_ARGS="-it"
+else
+ # The input device on kokoro is not a TTY, so -it does not work.
+ DOCKER_TTY_ARGS=
+fi
+
# running dockcross image without any arguments generates a wrapper
# scripts that can be used to run commands under the dockcross image
# easily.
# See https://github.com/dockcross/dockcross#usage for details
-docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
+docker run $DOCKER_TTY_ARGS --rm dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
chmod +x dockcross-manylinux2014-aarch64.sh
# the wrapper script has CRLF line endings and bash doesn't like that
diff --git a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh
index 7933be4..db2e776 100755
--- a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh
+++ b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh
@@ -1,21 +1,28 @@
#!/bin/bash
#
-# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
-# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+# Builds protobuf python including the C++ extension with aarch64 crosscompiler.
+# The outputs of this script are laid out so that we can later test them under an aarch64 emulator.
+# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image.
set -ex
PYTHON="/opt/python/cp38-cp38/bin/python"
./autogen.sh
-CXXFLAGS="-fPIC -g -O2" ./configure
+CXXFLAGS="-fPIC -g -O2" ./configure --host=aarch64
make -j8
+# create a simple shell wrapper that runs crosscompiled protoc under qemu
+echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh
+chmod ugo+x protoc_qemu_wrapper.sh
+
+# PROTOC variable is by build_py step that runs under ./python directory
+export PROTOC=../protoc_qemu_wrapper.sh
+
pushd python
-# TODO: currently this step relies on qemu being registered with binfmt_misc so that
-# aarch64 binaries are automatically run with an emulator. This works well once
-# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine.
+# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files.
${PYTHON} setup.py build_py
# when crosscompiling for aarch64, --plat-name needs to be set explicitly
diff --git a/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
new file mode 100755
index 0000000..f613202
--- /dev/null
+++ b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running
+# inside docker containers.
+
+set -ex
+
+# show pre-existing qemu registration
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
+
+# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that:
+# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278")
+# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below)
+#
+# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static
+# docker image to provide a new enough version of qemu-user-static and register it with
+# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"),
+# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc.
+# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator
+# binary to be accessible from the docker image we're emulating.
+# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient
+# to install qemu-user-static with the right flags.
+docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes
+
+# Print current qemu reqistration to make sure everything is setup correctly.
+cat /proc/sys/fs/binfmt_misc/qemu-aarch64
diff --git a/kokoro/linux/aarch64/test_python_aarch64.sh b/kokoro/linux/aarch64/test_python_aarch64.sh
index d5af6d6..1e2d570 100755
--- a/kokoro/linux/aarch64/test_python_aarch64.sh
+++ b/kokoro/linux/aarch64/test_python_aarch64.sh
@@ -5,6 +5,13 @@
# go to the repo root
cd $(dirname $0)/../../..
+if [[ -t 0 ]]; then
+ DOCKER_TTY_ARGS="-it"
+else
+ # The input device on kokoro is not a TTY, so -it does not work.
+ DOCKER_TTY_ARGS=
+fi
+
# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
@@ -16,4 +23,4 @@
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
-docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh
diff --git a/kokoro/linux/python_aarch64/build.sh b/kokoro/linux/python_aarch64/build.sh
new file mode 100755
index 0000000..2c67396
--- /dev/null
+++ b/kokoro/linux/python_aarch64/build.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_python_aarch64.sh
diff --git a/kokoro/linux/python_aarch64/continuous.cfg b/kokoro/linux/python_aarch64/continuous.cfg
new file mode 100644
index 0000000..dee4a47
--- /dev/null
+++ b/kokoro/linux/python_aarch64/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/python_aarch64/presubmit.cfg b/kokoro/linux/python_aarch64/presubmit.cfg
new file mode 100644
index 0000000..dee4a47
--- /dev/null
+++ b/kokoro/linux/python_aarch64/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/python_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
