Merge pull request #8495 from jtattermusch/run_ruby_aarch64_test
Run ruby aarch64 tests on the CI (with an emulator)
diff --git a/kokoro/linux/aarch64/test_csharp_aarch64.sh b/kokoro/linux/aarch64/test_csharp_aarch64.sh
new file mode 100755
index 0000000..91e30ce
--- /dev/null
+++ b/kokoro/linux/aarch64/test_csharp_aarch64.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+ DOCKER_TTY_ARGS="-it"
+else
+ # The input device on kokoro is not a TTY, so -it does not work.
+ DOCKER_TTY_ARGS=
+fi
+
+# First, build protobuf C# tests under x86_64 docker image
+# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
+# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
+CSHARP_BUILD_COMMAND="dotnet publish -c Release -f net50 csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:5.0.202-buster-slim bash -c "$CSHARP_BUILD_COMAND"
+
+# Use an actual aarch64 docker image to run protobuf C# tests with an emulator. "dotnet vstest" allows
+# running tests from a pre-built project.
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+# we just run map the user's home to a throwaway temporary directory
+CSHARP_TEST_COMMAND="dotnet vstest csharp/src/Google.Protobuf.Test/bin/Release/net50/publish/Google.Protobuf.Test.dll"
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "DOTNET_CLI_TELEMETRY_OPTOUT=true" -e "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work mcr.microsoft.com/dotnet/sdk:5.0.202-buster-slim-arm64v8 bash -c "$CSHARP_TEST_COMMAND"
diff --git a/kokoro/linux/aarch64/test_java_aarch64.sh b/kokoro/linux/aarch64/test_java_aarch64.sh
new file mode 100755
index 0000000..72f1c83
--- /dev/null
+++ b/kokoro/linux/aarch64/test_java_aarch64.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+if [[ -t 0 ]]; then
+ DOCKER_TTY_ARGS="-it"
+else
+ # The input device on kokoro is not a TTY, so -it does not work.
+ DOCKER_TTY_ARGS=
+fi
+
+# crosscompile protoc as we will later need it for the java build.
+# we build it under the dockcross/manylinux2014-aarch64 image so that the resulting protoc binary is compatible
+# with a wide range of linux distros (including any docker images we will use later to build and test java)
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/protoc_crosscompile_aarch64.sh
+
+# the command that will be used to build and test java under an emulator
+# * IsValidUtf8Test and DecodeUtf8Test tests are being skipped because that take very long under an emulator.
+TEST_JAVA_COMMAND="mvn --batch-mode -DskipTests install && mvn --batch-mode -Dtest='**/*Test, !**/*IsValidUtf8Test, !**/*DecodeUtf8Test' -DfailIfNoTests=false surefire:test"
+
+# use an actual aarch64 docker image (with a real aarch64 java and maven) to run build & test protobuf java under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+# we just run map the user's home to a throwaway temporary directory
+# * the JAVA_OPTS and MAVEN_CONFIG variables are being set mostly to silence warnings about non-existent home directory
+# and to avoid polluting the workspace.
+docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -e "JAVA_OPTS=-Duser.home=/home/fake-user" -e "MAVEN_CONFIG=/home/fake-user/.m2" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work arm64v8/maven:3.8-openjdk-11 bash -c "cd java && $TEST_JAVA_COMMAND"
diff --git a/kokoro/linux/csharp_aarch64/build.sh b/kokoro/linux/csharp_aarch64/build.sh
new file mode 100755
index 0000000..db3fc21
--- /dev/null
+++ b/kokoro/linux/csharp_aarch64/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_csharp_aarch64.sh
diff --git a/kokoro/linux/csharp_aarch64/continuous.cfg b/kokoro/linux/csharp_aarch64/continuous.cfg
new file mode 100644
index 0000000..df28ef3
--- /dev/null
+++ b/kokoro/linux/csharp_aarch64/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/csharp_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/csharp_aarch64/presubmit.cfg b/kokoro/linux/csharp_aarch64/presubmit.cfg
new file mode 100644
index 0000000..df28ef3
--- /dev/null
+++ b/kokoro/linux/csharp_aarch64/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/csharp_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_aarch64/build.sh b/kokoro/linux/java_aarch64/build.sh
new file mode 100755
index 0000000..ed5ad53
--- /dev/null
+++ b/kokoro/linux/java_aarch64/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This is the top-level script we give to Kokoro as the entry point for
+# running the "continuous" and "presubmit" jobs.
+
+set -ex
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Initialize any submodules.
+git submodule update --init --recursive
+
+kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh
+
+kokoro/linux/aarch64/test_java_aarch64.sh
diff --git a/kokoro/linux/java_aarch64/continuous.cfg b/kokoro/linux/java_aarch64/continuous.cfg
new file mode 100644
index 0000000..5db46d4
--- /dev/null
+++ b/kokoro/linux/java_aarch64/continuous.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/kokoro/linux/java_aarch64/presubmit.cfg b/kokoro/linux/java_aarch64/presubmit.cfg
new file mode 100644
index 0000000..5db46d4
--- /dev/null
+++ b/kokoro/linux/java_aarch64/presubmit.cfg
@@ -0,0 +1,11 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/java_aarch64/build.sh"
+timeout_mins: 120
+
+action {
+ define_artifacts {
+ regex: "**/sponge_log.xml"
+ }
+}
diff --git a/python/setup.py b/python/setup.py
index 696bde2..ff67b89 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -209,6 +209,18 @@
extra_compile_args = []
+ message_extra_link_args = None
+ api_implementation_link_args = None
+ if "darwin" in sys.platform:
+ if sys.version_info[0] == 2:
+ message_init_symbol = 'init_message'
+ api_implementation_init_symbol = 'init_api_implementation'
+ else:
+ message_init_symbol = 'PyInit__message'
+ api_implementation_init_symbol = 'PyInit__api_implementation'
+ message_extra_link_args = ['-Wl,-exported_symbol,_%s' % message_init_symbol]
+ api_implementation_link_args = ['-Wl,-exported_symbol,_%s' % api_implementation_init_symbol]
+
if sys.platform != 'win32':
extra_compile_args.append('-Wno-write-strings')
extra_compile_args.append('-Wno-invalid-offsetof')
@@ -259,6 +271,7 @@
include_dirs=[".", "../src"],
libraries=libraries,
extra_objects=extra_objects,
+ extra_link_args=message_extra_link_args,
library_dirs=['../src/.libs'],
extra_compile_args=extra_compile_args,
),
@@ -266,6 +279,7 @@
"google.protobuf.internal._api_implementation",
glob.glob('google/protobuf/internal/api_implementation.cc'),
extra_compile_args=extra_compile_args + ['-DPYTHON_PROTO2_CPP_IMPL_V2'],
+ extra_link_args=api_implementation_link_args,
),
])
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
