Migrate non-Linux C++ tests to GHA actions
PiperOrigin-RevId: 506683178
diff --git a/.github/actions/bazel-docker/action.yml b/.github/actions/bazel-docker/action.yml
index 4077574..c429163 100644
--- a/.github/actions/bazel-docker/action.yml
+++ b/.github/actions/bazel-docker/action.yml
@@ -11,7 +11,7 @@
description: "The docker image to use"
type: string
bazel-cache:
- required: false
+ required: true
description: >
A unique path for the Bazel cache. This will trigger the generation
of a BAZEL_CACHE environment variable inside the container that provides
@@ -31,35 +31,25 @@
steps:
- name: Authenticate
id: auth
- uses: ./.github/actions/internal/docker-auth
+ uses: ./.github/actions/internal/gcloud-auth
with:
credentials: ${{ inputs.credentials }}
+ - name: Setup Runner
+ uses: ./.github/actions/internal/setup-runner
+
+ - name: Setup Bazel
+ id: bazel
+ uses: ./.github/actions/internal/bazel-setup
+ with:
+ credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
+ bazel-cache: ${{ inputs.bazel-cache }}
+
- name: Validate inputs
- if: ${{ inputs.bash && inputs.bazel}}
+ if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
shell: bash
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
- - name: Initialize Bazel flags
- shell: bash
- run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
-
- - name: Configure Bazel caching
- shell: bash
- # Skip bazel cache for local act runs due to issue with credential files
- # and nested docker images
- if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
- run: >
- echo "BAZEL_FLAGS=$BAZEL_FLAGS
- --google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
- --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
-
- - name: Configure Bazel cache writing
- shell: bash
- # External PRs should never write to our caches.
- if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
- run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
-
- name: Run Bash Docker
uses: ./.github/actions/internal/docker-run
if: ${{ inputs.bash }}
@@ -73,4 +63,4 @@
if: ${{ !inputs.bash }}
with:
image: ${{ inputs.image }}
- command: ${{ inputs.bazel }} $BAZEL_FLAGS
+ command: ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml
new file mode 100644
index 0000000..8501615
--- /dev/null
+++ b/.github/actions/bazel/action.yml
@@ -0,0 +1,75 @@
+name: 'Docker Bazel Run'
+description: 'Run a Bazel-based docker image for Protobuf CI testing'
+inputs:
+ credentials:
+ required: true
+ description: The GCP credentials to use for reading the docker image
+ type: string
+ bazel-cache:
+ required: true
+ description: >
+ A unique path for the Bazel cache. This will trigger the generation
+ of a BAZEL_CACHE environment variable inside the container that provides
+ the appropriate flags for any bazel command.
+ type: string
+ version:
+ required: false
+ description: A pinned Bazel version to use
+ default: '5.1.1'
+ type: string
+ bazel:
+ required: false
+ description: The Bazel command to run
+ type: string
+ bash:
+ required: false
+ description: >
+ A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be
+ available to use for bazel runs.
+ type: string
+
+runs:
+ using: 'composite'
+ steps:
+ - name: Authenticate
+ id: auth
+ uses: ./.github/actions/internal/gcloud-auth
+ with:
+ credentials: ${{ inputs.credentials }}
+
+ - name: Setup Runner
+ uses: ./.github/actions/internal/setup-runner
+
+ - name: Setup Bazel
+ id: bazel
+ uses: ./.github/actions/internal/bazel-setup
+ with:
+ credentials-file: ${{ steps.auth.outputs.credentials-file }}
+ bazel-cache: ${{ inputs.bazel-cache }}
+
+ - name: Validate inputs
+ if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
+ shell: bash
+ run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
+
+ - name: Output Bazel version
+ env:
+ USE_BAZEL_VERSION: ${{ inputs.version }}
+ shell: bash
+ run: bazelisk version
+
+ - name: Run Bash
+ env:
+ USE_BAZEL_VERSION: ${{ inputs.version }}
+ if: ${{ inputs.bash }}
+ run: ${{ inputs.bash }}
+ shell: bash
+
+ - name: Run Bazel
+ env:
+ USE_BAZEL_VERSION: ${{ inputs.version }}
+ if: ${{ !inputs.bash }}
+ run: >
+ bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
+ ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }}
+ shell: bash
diff --git a/.github/actions/non-bazel-docker/action.yml b/.github/actions/docker/action.yml
similarity index 93%
rename from .github/actions/non-bazel-docker/action.yml
rename to .github/actions/docker/action.yml
index 23e502a..93bab6b 100644
--- a/.github/actions/non-bazel-docker/action.yml
+++ b/.github/actions/docker/action.yml
@@ -17,6 +17,9 @@
runs:
using: 'composite'
steps:
+ - name: Setup Runner
+ uses: ./.github/actions/internal/setup-runner
+
- name: Update stale files using Bazel
uses: ./.github/actions/bazel-docker
with:
diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml
new file mode 100644
index 0000000..28d916a
--- /dev/null
+++ b/.github/actions/internal/bazel-setup/action.yml
@@ -0,0 +1,59 @@
+name: Setup Bazel
+description: Setup a Bazel environment for Protobuf CI testing
+inputs:
+ credentials-file:
+ required: true
+ description: The GCP credentials file to use for caching
+ type: string
+ bazel-cache:
+ required: true
+ description: A unique path for the Bazel cache.
+ type: string
+
+outputs:
+ bazel-flags:
+ description: Bazel flags that should be sent to all Bazel invocations
+ value: ${{ steps.output.outputs.bazel-flags }}
+ bazel-startup-flags:
+ description: Bazel startup flags that should be sent to all Bazel invocations
+ value: ${{ steps.output.outputs.bazel-startup-flags }}
+
+runs:
+ using: 'composite'
+ steps:
+ - name: Initialize Windows startup flags
+ if: runner.os == 'Windows'
+ shell: bash
+ run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV
+
+ - name: Initialize Bazel flags
+ shell: bash
+ run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
+
+ - name: Initialize Windows-specific Bazel flags
+ if: runner.os == 'Windows'
+ shell: bash
+ run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV
+
+ - name: Configure Bazel caching
+ # Skip bazel cache for local act runs due to issue with credential files
+ # and nested docker images
+ if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
+ shell: bash
+ run: >
+ echo "BAZEL_FLAGS=$BAZEL_FLAGS
+ --google_credentials='${{ inputs.credentials-file }}'
+ --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
+
+ - name: Configure Bazel cache writing
+ # External runs should never write to our caches.
+ if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
+ shell: bash
+ run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
+
+ - name: Output Bazel flags
+ id: output
+ shell: bash
+ run: |
+ echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT
+ echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT
diff --git a/.github/actions/internal/docker-run/action.yml b/.github/actions/internal/docker-run/action.yml
index 9e29418..1cc1489 100644
--- a/.github/actions/internal/docker-run/action.yml
+++ b/.github/actions/internal/docker-run/action.yml
@@ -22,6 +22,10 @@
runs:
using: 'composite'
steps:
+ - name: Authenticate for GAR use
+ shell: bash
+ run: gcloud auth configure-docker -q us-docker.pkg.dev
+
- name: Setup QEMU for possible emulation
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
diff --git a/.github/actions/internal/docker-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml
similarity index 76%
rename from .github/actions/internal/docker-auth/action.yml
rename to .github/actions/internal/gcloud-auth/action.yml
index 482ce7a..8c0f2ca 100644
--- a/.github/actions/internal/docker-auth/action.yml
+++ b/.github/actions/internal/gcloud-auth/action.yml
@@ -1,9 +1,9 @@
-name: 'Authenticate Docker'
+name: 'Authenticate for GCP'
description: 'Authenticate a workflow for Protobuf CI testing'
inputs:
credentials:
required: true
- description: "The GCP credentials to use for reading the docker image"
+ description: "The GCP credentials to use for GCP"
type: string
outputs:
@@ -24,6 +24,3 @@
- name: Use gcloud CLI
shell: bash
run: gcloud info
- - name: Authenticate for GAR use
- shell: bash
- run: gcloud auth configure-docker -q us-docker.pkg.dev
diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml
new file mode 100644
index 0000000..5c4f783
--- /dev/null
+++ b/.github/actions/internal/setup-runner/action.yml
@@ -0,0 +1,17 @@
+name: Setup CI Runner
+# TODO(b/267357823) Consider moving this to it's own repository and include
+# a call to actions/checkout.
+description: Setup any platform-specific adjustments we need to make for CI
+runs:
+ using: 'composite'
+ steps:
+ - name: Fix Windows line breaks
+ if: runner.os == 'Windows'
+ shell: bash
+ run: find . -type f -print0 | xargs -0 d2u 2>/dev/null
+
+ - name: Install bazelrc files
+ shell: bash
+ run: |
+ cp ci/*.bazelrc .
+ cp -f ${{ runner.os }}.bazelrc .bazelrc
diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
index 5909f9d..03d3cb3 100644
--- a/.github/workflows/test_cpp.yml
+++ b/.github/workflows/test_cpp.yml
@@ -88,8 +88,33 @@
submodules: recursive
ref: ${{ inputs.safe-checkout }}
- name: Run tests
- uses: ./.github/actions/non-bazel-docker
+ uses: ./.github/actions/docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: ${{ matrix.command }}
+
+ non-linux:
+ strategy:
+ fail-fast: false # Don't cancel all jobs if one fails.
+ matrix:
+ include:
+ - name: MacOS
+ os: macos-12
+ bazel: //src/...
+ - name: Windows
+ os: windows-2019
+ bazel: //src/... @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance
+ name: ${{ matrix.name }} Bazel
+ runs-on: ${{ matrix.os }}
+ steps:
+ - name: Checkout pending changes
+ uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+ with:
+ ref: ${{ inputs.safe-checkout }}
+ - name: Run tests
+ uses: ./.github/actions/bazel
+ with:
+ credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+ bazel: test ${{ matrix.bazel }}
+ bazel-cache: cpp_${{ matrix.os }}
diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml
index 0bf0ceb..7023bf5 100644
--- a/.github/workflows/test_php.yml
+++ b/.github/workflows/test_php.yml
@@ -43,7 +43,7 @@
submodules: recursive
ref: ${{ inputs.safe-checkout }}
- name: Run tests
- uses: ./.github/actions/non-bazel-docker
+ uses: ./.github/actions/docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
diff --git a/ci/Linux.bazelrc b/ci/Linux.bazelrc
new file mode 100644
index 0000000..d5dcf5d
--- /dev/null
+++ b/ci/Linux.bazelrc
@@ -0,0 +1,3 @@
+import common.bazelrc
+
+build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14
diff --git a/ci/Windows.bazelrc b/ci/Windows.bazelrc
new file mode 100644
index 0000000..427e923
--- /dev/null
+++ b/ci/Windows.bazelrc
@@ -0,0 +1,2 @@
+import common.bazelrc
+
diff --git a/ci/common.bazelrc b/ci/common.bazelrc
new file mode 100644
index 0000000..c1d90ed
--- /dev/null
+++ b/ci/common.bazelrc
@@ -0,0 +1,33 @@
+build:dbg --compilation_mode=dbg
+
+build:opt --compilation_mode=opt
+
+build:san-common --config=dbg --strip=never --copt=-O0 --copt=-fno-omit-frame-pointer
+
+build:asan --config=san-common --copt=-fsanitize=address --linkopt=-fsanitize=address
+build:asan --copt=-DADDRESS_SANITIZER=1
+# ASAN hits ODR violations with shared linkage due to rules_proto.
+build:asan --dynamic_mode=off
+
+build:msan --config=san-common --copt=-fsanitize=memory --linkopt=-fsanitize=memory
+build:msan --copt=-fsanitize-memory-track-origins
+build:msan --copt=-fsanitize-memory-use-after-dtor
+build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
+build:msan --copt=-DMEMORY_SANITIZER=1
+
+# Use our instrumented LLVM libc++ in Kokoro.
+build:kokoro-msan --config=msan
+build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
+build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
+build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
+
+
+build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread
+build:tsan --copt=-DTHREAD_SANITIZER=1
+
+build:ubsan --config=san-common --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined
+build:ubsan --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
+build:ubsan --copt=-DUNDEFINED_SANITIZER=1
+# Workaround for the fact that Bazel links with $CC, not $CXX
+# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748
+build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc
new file mode 100644
index 0000000..d5dcf5d
--- /dev/null
+++ b/ci/macOS.bazelrc
@@ -0,0 +1,3 @@
+import common.bazelrc
+
+build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14
diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc
index 1cacac0..734543e 100644
--- a/src/google/protobuf/io/zero_copy_stream_unittest.cc
+++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc
@@ -731,9 +731,11 @@
// Verifies that outputs up to kint32max can be created.
TEST_F(IoTest, LargeOutput) {
- // Filter out this test on 32-bit architectures and tsan builds.
+ // Filter out this test on 32-bit architectures and builds where our test
+ // infrastructure can't handle it.
if(sizeof(void*) < 8) return;
-#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER)
+#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && \
+ !defined(_MSC_VER)
std::string str;
StringOutputStream output(&str);
void* unused_data;
