Migrate remaining macOS tests to GHA actions PiperOrigin-RevId: 506744609
diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml index 8501615..d04cf8b 100644 --- a/.github/actions/bazel/action.yml +++ b/.github/actions/bazel/action.yml
@@ -52,24 +52,22 @@ shell: bash run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 + - name: Pin Bazel version + shell: bash + run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV + - name: Output Bazel version - env: - USE_BAZEL_VERSION: ${{ inputs.version }} shell: bash run: bazelisk version - name: Run Bash - env: - USE_BAZEL_VERSION: ${{ inputs.version }} if: ${{ inputs.bash }} run: ${{ inputs.bash }} shell: bash - name: Run Bazel - env: - USE_BAZEL_VERSION: ${{ inputs.version }} if: ${{ !inputs.bash }} - run: > + run: >- bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }} shell: bash
diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml index 28d916a..c4059b3 100644 --- a/.github/actions/internal/bazel-setup/action.yml +++ b/.github/actions/internal/bazel-setup/action.yml
@@ -21,6 +21,10 @@ runs: using: 'composite' steps: + - name: Initialize BAZEL environment variable + shell: bash + run: echo "BAZEL=bazelisk" >> $GITHUB_ENV + - name: Initialize Windows startup flags if: runner.os == 'Windows' shell: bash @@ -40,9 +44,9 @@ # and nested docker images if: ${{ inputs.bazel-cache && !github.event.act_local_test }} shell: bash - run: > + run: >- echo "BAZEL_FLAGS=$BAZEL_FLAGS - --google_credentials='${{ inputs.credentials-file }}' + --google_credentials=${{ inputs.credentials-file }} --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV - name: Configure Bazel cache writing
diff --git a/.github/actions/internal/gcloud-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml index 8c0f2ca..5100a4c 100644 --- a/.github/actions/internal/gcloud-auth/action.yml +++ b/.github/actions/internal/gcloud-auth/action.yml
@@ -9,7 +9,7 @@ outputs: credentials-file: description: "Credentials file generated for GCP" - value: ${{ steps.auth.outputs.credentials_file_path }} + value: ${{ steps.output.outputs.credentials-file }} runs: using: 'composite' @@ -24,3 +24,18 @@ - name: Use gcloud CLI shell: bash run: gcloud info + + - name: Store credentials path + if: ${{ runner.os != 'Windows' }} + shell: bash + run: echo "credentials-file=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV + - name: Store credentials path (Windows) + if: ${{ runner.os == 'Windows' }} + # Bash commands in windows don't like the backslash in the file path. + shell: bash + run: echo "credentials-file='${{ steps.auth.outputs.credentials_file_path }}'" >> $GITHUB_ENV + + - name: Output credentials file + id: output + shell: bash + run: echo "credentials-file=${{ env.credentials-file }}" >> $GITHUB_OUTPUT
diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml index 5c4f783..64afdc3 100644 --- a/.github/actions/internal/setup-runner/action.yml +++ b/.github/actions/internal/setup-runner/action.yml
@@ -1,6 +1,6 @@ name: Setup CI Runner -# TODO(b/267357823) Consider moving this to it's own repository and include -# a call to actions/checkout. +# TODO(b/267357823) Consider moving this to it's own repository so we can +# include the call to actions/checkout. description: Setup any platform-specific adjustments we need to make for CI runs: using: 'composite'
diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml new file mode 100644 index 0000000..7cb54e6 --- /dev/null +++ b/.github/workflows/test_objectivec.yml
@@ -0,0 +1,45 @@ +name: Objective-c Tests + +on: + workflow_call: + inputs: + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string + +jobs: + macos: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + include: + - name: OS X + config: osx + flags: --core-only --skip-xcode-ios --skip-xcode-tvos + - name: iOS Debug + config: ios_debug + flags: --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-release + - name: iOS Release + config: ios_release + flags: --core-only --skip-xcode-osx --skip-xcode-tvos --skip-objc-conformance --skip-xcode-debug + + name: ${{ matrix.name}} + runs-on: macos-12 + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + + - name: Select pinned Xcode version + uses: maxim-lobanov/setup-xcode@9a697e2b393340c3cacd97468baa318e4c883d98 # v1.5.1 + with: + xcode-version: '14.1' + + - name: Run tests + uses: ./.github/actions/bazel + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: objectivec_macos/${{ matrix.config }} + bash: objectivec/DevTools/full_mac_build.sh ${{ matrix.flags }}
diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 7023bf5..b492b04 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml
@@ -34,7 +34,7 @@ version: 8.0.5-dbg command: composer test_valgrind - name: Linux PHP ${{ matrix.name}} + name: Linux ${{ matrix.name}} runs-on: ubuntu-latest steps: - name: Checkout pending changes @@ -48,3 +48,54 @@ image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} + + macos: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + version: ['7.4', '8.0'] + + name: MacOS PHP ${{ matrix.version }} + runs-on: macos-12 + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Install dependencies + run: brew install coreutils # For sha256sum + + - name: Pin PHP version + uses: shivammathur/setup-php@d30ad8b1843ace22e6698ab99bbafaa747b6bd0d # 2.24.0 + with: + php-version: ${{ matrix.version }} + + - name: Check PHP version + run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1) + + - name: Run tests + uses: ./.github/actions/bazel + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: php_macos/${{ matrix.version }} + bash: | + pushd php + rm -rf vendor + php -v + php -m + composer update + composer test_c + popd + + - name: Run conformance tests + uses: ./.github/actions/bazel + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: php_macos/${{ matrix.version }} + bash: >- + bazelisk $BAZEL_STARTUP_FLAGS \ + test $BAZEL_FLAGS \ + --action_env=PATH --test_env=PATH \ + //php:conformance_test_c
diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 301b089..e7b76a4 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml
@@ -43,3 +43,53 @@ credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION + + + macos: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + type: [ Pure, C++] + # TODO(b/267550843) Consider expanding this set of versions. + version: [ "3.10" ] + include: + - type: Pure + targets: //python/... @upb//python/... //python:python_version + - type: C++ + targets: //python/... //python:python_version + flags: --define=use_fast_cpp_protos=true + + name: MacOS ${{ matrix.type }} ${{ matrix.version }} + runs-on: macos-12 + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Pin Python version + uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0 + with: + python-version: ${{ matrix.version }} + cache: pip + + - name: Validate version + run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1) + + - name: Create and start virtual environment + run: | + python3 -m venv venv + source venv/bin/activate + + - name: Run tests + uses: ./.github/actions/bazel + env: + KOKORO_PYTHON_VERSION: ${{ matrix.version }} + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }} + bazel: >- + test ${{ matrix.targets }} ${{ matrix.flags }} + --test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }} + --macos_minimum_os=10.9
diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index 44ee193..54194e6 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml
@@ -39,6 +39,37 @@ bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION + macos: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + # This is the full set of versions we support on MacOS. + version: [ "2.7", "3.0", "3.1", "3.2" ] + + name: MacOS Ruby ${{ matrix.version }} + runs-on: macos-12 + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Pin Ruby version + uses: ruby/setup-ruby@ee26e27437bde475b19a6bf8cb73c9fa658876a2 # v1.134.0 + with: + ruby-version: ${{ matrix.version }} + + - name: Validate version + run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1) + + - name: Run tests + uses: ./.github/actions/bazel + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel-cache: ruby_macos/${{ matrix.version }} + bazel: test //ruby/... --test_env=KOKORO_RUBY_VERSION=${{ matrix.version }} + test_ruby_gems: strategy: fail-fast: false
diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml index fa4f5d4..f4b2f2a 100644 --- a/.github/workflows/test_runner.yml +++ b/.github/workflows/test_runner.yml
@@ -128,3 +128,11 @@ with: safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} secrets: inherit + + objectivec: + name: Objective-c + needs: [check-tag] + uses: ./.github/workflows/test_objectivec.yml + with: + safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} + secrets: inherit
diff --git a/generate_descriptor_proto.sh b/generate_descriptor_proto.sh index 91c1217..43a060b 100755 --- a/generate_descriptor_proto.sh +++ b/generate_descriptor_proto.sh
@@ -62,7 +62,7 @@ PROTOC=$BOOTSTRAP_PROTOC BOOTSTRAP_PROTOC="" else - bazel build $@ //:protoc + ${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-} build $@ //:protoc ${BAZEL_FLAGS:-} if test $? -ne 0; then echo "Failed to build protoc." exit 1
diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index 4fed4f5..b1e6fb1 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh
@@ -8,11 +8,11 @@ # Some base locations. readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") readonly ProtoRootDir="${ScriptDir}/../.." -readonly BazelFlags="--announce_rc --macos_minimum_os=10.9 \ - $(${ScriptDir}/../../kokoro/common/bazel_flags.sh)" +readonly BazelFlags="${BAZEL_FLAGS:---announce_rc --macos_minimum_os=10.9 \ + $(${ScriptDir}/../../kokoro/common/bazel_flags.sh)}" # Invoke with BAZEL=bazelisk to use that instead. -readonly BazelBin="${BAZEL:=bazel}" +readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}" printUsage() { NAME=$(basename "${0}") @@ -61,7 +61,6 @@ echo "========================================================================" } -BAZEL=bazel DO_CLEAN=no REGEN_DESCRIPTORS=no CORE_ONLY=no @@ -132,7 +131,7 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then header "Cleaning" - "${BazelBin}" clean + ${BazelBin} clean if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then XCODEBUILD_CLEAN_BASE_IOS=( xcodebuild @@ -181,16 +180,16 @@ if [[ "${CORE_ONLY}" == "yes" ]] ; then header "Building core Only" - "${BazelBin}" build //:protoc //:protobuf //:protobuf_lite $BazelFlags + ${BazelBin} build //:protoc //:protobuf //:protobuf_lite $BazelFlags else header "Building" # Can't issue these together, when fully parallel, something sometimes chokes # at random. - "${BazelBin}" test //src/... $BazelFlags + ${BazelBin} test //src/... $BazelFlags fi # Ensure the WKT sources checked in are current. -BAZEL="${BazelBin}" objectivec/generate_well_known_types.sh --check-only $BazelFlags +objectivec/generate_well_known_types.sh --check-only $BazelFlags header "Checking on the ObjC Runtime Code" # Some of the kokoro machines don't have python3 yet, so fall back to python if need be. @@ -317,7 +316,7 @@ if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then header "Running ObjC Conformance Tests" - "${BazelBin}" test //objectivec:conformance_test $BazelFlags + ${BazelBin} test //objectivec:conformance_test $BazelFlags fi echo ""
diff --git a/objectivec/generate_well_known_types.sh b/objectivec/generate_well_known_types.sh index 83a341e..7e23a99 100755 --- a/objectivec/generate_well_known_types.sh +++ b/objectivec/generate_well_known_types.sh
@@ -10,7 +10,7 @@ readonly ProtoRootDir="${ObjCDir}/.." # Invoke with BAZEL=bazelisk to use that instead. -readonly BazelBin="${BAZEL:=bazel}" +readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS:-}" # Flag for continuous integration to check that everything is current. CHECK_ONLY=0 @@ -30,7 +30,7 @@ fi # Make sure the compiler is current. -"${BazelBin}" build //:protoc $@ +${BazelBin} build //:protoc $@ cd src declare -a RUNTIME_PROTO_FILES=( \
diff --git a/php/generate_descriptor_protos.sh b/php/generate_descriptor_protos.sh index 0e5be35..a9af40c 100755 --- a/php/generate_descriptor_protos.sh +++ b/php/generate_descriptor_protos.sh
@@ -9,7 +9,7 @@ PROTOC=$(realpath protoc) fi if [ ! -f $PROTOC ]; then - bazel build -c opt //:protoc + ${BAZEL:-bazel} $BAZEL_STARTUP_FLAGS build -c opt //:protoc $BAZEL_FLAGS PROTOC=$(realpath bazel-bin/protoc) fi
diff --git a/php/generate_test_protos.sh b/php/generate_test_protos.sh index 6d06f95..51df6f8 100755 --- a/php/generate_test_protos.sh +++ b/php/generate_test_protos.sh
@@ -8,7 +8,7 @@ PROTOC=$(pwd)/protoc fi if [ ! -f $PROTOC ]; then - bazel build -c opt //:protoc + ${BAZEL:-bazel} $BAZEL_STARTUP_FLAGS build -c opt //:protoc $BAZEL_FLAGS PROTOC=$(pwd)/bazel-bin/protoc fi