Clean up test infrastructure (#41880)
diff --git a/.cirrus.yml b/.cirrus.yml index 9eb90de..3b380f3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml
@@ -1,18 +1,41 @@ -container: - image: gcr.io/flutter-cirrus/build-flutter-image:latest +# CIRRUS CONFIGURATION FILE +use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' + +environment: + # For details about environment variables used in Cirrus, including how encrypted variables work, + # see https://cirrus-ci.org/guide/writing-tasks/#environment-variables + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + # We change Flutter's directory to include a space in its name (see $CIRRUS_WORKING_DIR) so that + # we constantly test path names with spaces in them. The FLUTTER_SDK_PATH_WITH_SPACE variable must + # therefore have a space in it. + FLUTTER_SDK_PATH_WITH_SPACE: "flutter sdk" + # We force BOT to true so that all our tools know we're in a CI environment. This avoids any + # dependency on precisely how Cirrus is detected by our tools. + BOT: "true" + # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they might include non-ASCII + # characters which makes Gradle crash. See: https://github.com/flutter/flutter/issues/24935 + # TODO(amirha): remove once we've migrated to newer Gradle + CIRRUS_CHANGE_MESSAGE: "" + CIRRUS_COMMIT_MESSAGE: "" + +# LINUX SHARDS task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - # Name the SDK directory to include a space so that we constantly - # test path names with spaces in them. - CIRRUS_WORKING_DIR: "/tmp/flutter sdk" + container: + image: gcr.io/flutter-cirrus/build-flutter-image:latest + cpu: $CPU + memory: $MEMORY + environment: + # We shrink our default resource requirement as much as possible because that way we are more + # likely to get scheduled. We require 4G of RAM because most of the shards (all but one as of + # October 2019) just get OOM-killed with less. Some shards may need more. When increasing the + # requirements for select shards, please leave a comment on those shards saying when you + # increased the requirements, what numbers you tried, and what the results were. + CPU: 1 # 0.1-8 without compute credits, 0.1-30 with (yes, you can go fractional) + MEMORY: 4G # 256M-24G without compute credits, 256M-90G with + CIRRUS_WORKING_DIR: "/tmp/$FLUTTER_SDK_PATH_WITH_SPACE" PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH" ANDROID_SDK_ROOT: "/opt/android_sdk" - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work pub_cache: folder: $HOME/.pub-cache fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR" @@ -22,345 +45,302 @@ artifacts_cache: folder: bin/cache/artifacts fingerprint_script: echo $OS; cat bin/internal/*.version - setup_script: ./dev/bots/cirrus_setup.sh + setup_script: + - date + - git clean -xffd + - git fetch origin + - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works. + - flutter config --no-analytics + - flutter doctor -v + - flutter update-packages + - ./dev/bots/accept_android_sdk_licenses.sh + - date + on_failure: + failure_script: + - date + - which flutter matrix: - - name: docs - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'" - env: - SHARD: docs + - name: analyze-linux # linux-only + # environment: + # Empirically, the analyze-linux shard runs surprisingly fast (under 15 minutes) with just 1 + # CPU and 4G RAM as of October 2019. It fails with less than 4G though. + script: + - dart --enable-asserts ./dev/bots/analyze.dart + + - name: framework_tests-widgets-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # We use 3 CPUs because that's the minimum required to get framework_tests-widgets-linux + # running fast enough that it is not the long pole, as of OCtober 2019. + CPU: 3 + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] + script: + - ./dev/bots/download_goldctl.sh + - dart --enable-asserts ./dev/bots/test.dart + + - name: framework_tests-libraries-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # We use 3 CPUs because that's the minimum required to get the + # framework_tests-libraries-linux shard running fast enough that it is not the long pole, as + # of October 2019. + CPU: 3 + GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" + GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] + script: + - ./dev/bots/download_goldctl.sh + - dart --enable-asserts ./dev/bots/test.dart + + - name: framework_tests-misc-linux + # this includes the tests for directories in dev/ + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # We use 3 CPUs because that's the minimum required to get framework_tests-misc-linux + # running fast enough that it is not the long pole, as of October 2019. + CPU: 3 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-general-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the tool_tests-general-linux shard got faster with more CPUs up to 4 + # CPUs, and needed at least 8G of RAM to not run out of memory. + CPU: 4 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-commands-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the tool_tests-commands-linux shard got faster with more CPUs up to 6 + # CPUs, and needed at least 8G of RAM to not run out of memory. + CPU: 6 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-integration-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the tool_tests-integration-linux shard got faster with more CPUs up to + # 6 CPUs, and needed at least 8G of RAM to not run out of memory. + CPU: 6 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_coverage-linux # linux-only + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**/*.dart', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the tool_coverage-linux shard needed at least 12G of RAM to run without + # getting OOM-killed, and even 8 CPUs took 25 minutes. + CPU: 8 + MEMORY: 12G + CODECOV_TOKEN: ENCRYPTED[7c76a7f8c9264f3b7f3fd63fcf186f93c62c4dfe43ec288861c2f506d456681032b89efe7b7a139c82156350ca2c752c] + script: + - dart --enable-asserts ./dev/bots/test.dart + - bash <(curl -s https://codecov.io/bash) -c -f packages/flutter_tools/coverage/lcov.info -F flutter_tool + + - name: web_tests-0-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: web_tests-1-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: web_tests-2-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: web_tests-3-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: web_tests-4-linux + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: web_tests-5_last-linux # last Web shard must end with _last + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # As of October 2019, the Web shards needed more than 6G of RAM. + CPU: 2 + MEMORY: 8G + allow_failures: true + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: build_tests-linux + environment: + # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed. + # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2 + # and 4) reduced that to just over 20 minutes. 6G was enough not to get OOM-killed. + CPU: 2 + MEMORY: 6G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-0-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-1-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-2-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-3-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-4-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-5_last-linux + environment: + # Some of the host-only devicelab tests are pretty involved and need a lot of RAM. + CPU: 2 + MEMORY: 8G + script: + - dart --enable-asserts ./dev/bots/test.dart + + # TODO(ianh): name: add_to_app_tests-linux + + - name: docs-linux # linux-only + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: + # Empirically, as of October 2019, the docs-linux shard took about 30 minutes when run with + # 1 CPU and 4G of RAM. 2 CPUs reduced that to 20 minutes, more CPUs did not improve matters. + CPU: 2 # For uploading master docs to Firebase master branch staging site FIREBASE_MASTER_TOKEN: ENCRYPTED[eb768d18798fdc5abfe09b224e1724c4d82831d715ccf90df2c79d618c317216cbd99493278361f6fe7948b409b603f0] # For uploading beta docs to Firebase public live site FIREBASE_PUBLIC_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92] - docs_script: ./dev/bots/docs.sh - - name: deploy_gallery - depends_on: - - docs - - analyze - - build_tests-linux - env: - SHARD: deploy_gallery - GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346] - ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0f2aca35f05b26add5d9edea2a7449341269a2b7e22d5c667f876996e2e8bc44ff1369431ebf73b7c5581fd95d0e5902] - test_script: - # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they - # might include non-ASCII characters which makes Gradle crash. - # See: https://github.com/flutter/flutter/issues/24935 - # This is a temporary workaround until we figure how to properly configure - # a UTF8 locale on Cirrus (or until the Gradle bug is fixed). - # TODO(amirh): Set the locale to UTF8. - - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt - - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt - - export CIRRUS_CHANGE_MESSAGE="" - - export CIRRUS_COMMIT_MESSAGE="" - - ./dev/bots/deploy_gallery.sh - - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt` - - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt` - - name: analyze - test_script: - - dart --enable-asserts ./dev/bots/analyze.dart - - name: bots_tests-linux - skip: "!changesInclude('dev/bots/**')" - test_script: - - (cd ./dev/bots && pub run test) - container: - cpu: 4 - memory: 12G - - name: tests_widgets-linux - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: widgets - GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" - GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: ./dev/bots/download_goldctl.sh - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: tests_framework_other-linux - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: framework_other - GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" - GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: ./dev/bots/download_goldctl.sh - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: tests_extras-linux - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: extras - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - # all of the tests except the ones in test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests_general-linux - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: general - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: tool_tests_commands-linux - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: commands - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - # all of the tests in test/integration for packages/flutter_tools - - name: tool_tests_integration-linux - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: integration - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: tool_coverage-linux - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - CODECOV_TOKEN: ENCRYPTED[7c76a7f8c9264f3b7f3fd63fcf186f93c62c4dfe43ec288861c2f506d456681032b89efe7b7a139c82156350ca2c752c] - SHARD: tool_coverage - test_script: - - dart --enable-asserts ./dev/bots/test.dart - - bash <(curl -s https://codecov.io/bash) -c -s ./packages/flutter_tools/coverage/ -f '*.lcov.info' -F flutter_tool - container: - cpu: 8 - memory: 24G - - name: web_tests-linux-shard-0 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 0 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: web_tests-linux-shard-1 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 1 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: web_tests-linux-shard-2 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 2 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: web_tests-linux-shard-3 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 3 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: web_tests-linux-shard-4 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 4 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: web_tests-linux-shard-5 - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - env: - SHARD: web_tests - WEB_SHARD: 5 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: build_tests-linux - env: - SHARD: build_tests - - name: integration_tests-linux - env: - SHARD: integration_tests - test_script: - # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they - # might include non-ASCII characters which makes Gradle crash. - # See: https://github.com/flutter/flutter/issues/24935 - # This is a temporary workaround until we figure how to properly configure - # a UTF8 locale on Cirrus (or until the Gradle bug is fixed). - # TODO(amirh): Set the locale to UTF8. - - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt - - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt - - export CIRRUS_CHANGE_MESSAGE="" - - export CIRRUS_COMMIT_MESSAGE="" - - dart --enable-asserts ./dev/bots/test.dart - - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt` - - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt` - container: - cpu: 4 - memory: 12G - - name: gradle_tests-linux-shard-1 - env: - SHARD: integration_tests - SUBSHARD: gradle1 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_tests-linux-shard-2 - env: - SHARD: integration_tests - SUBSHARD: gradle2 - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_embedding_v2_tests-linux-shard-1 - env: - SHARD: integration_tests - SUBSHARD: gradle1 - ENABLE_ANDROID_EMBEDDING_V2: 'true' - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_embedding_v2_tests-linux-shard-2 - env: - SHARD: integration_tests - SUBSHARD: gradle2 - ENABLE_ANDROID_EMBEDDING_V2: 'true' - test_script: - - dart --enable-asserts ./dev/bots/test.dart - container: - cpu: 4 - memory: 12G - - name: release_smoke_tests - env: - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec] - test_script: - - echo "$CIRRUS_CHANGE_MESSAGE" > /tmp/cirrus_change_message.txt - - echo "$CIRRUS_COMMIT_MESSAGE" > /tmp/cirrus_commit_message.txt - - export CIRRUS_CHANGE_MESSAGE="" - - export CIRRUS_COMMIT_MESSAGE="" - - ./dev/bots/firebase_testlab.sh - - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt` - - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt` + script: + - ./dev/bots/docs.sh + - name: customer_testing-linux + # environment: + # Empirically, this shard runs fine at 1 CPU and 4G RAM as of October 2019. We will probably + # want to grow this container when we invite people to add their tests in large numbers. script: - rm -rf bin/cache/pkg/tests - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test -task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - windows_container: - image: cirrusci/android-sdk:28-windowsservercore-2019 - os_version: 2019 - cpu: 4 - env: - CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk" - PATH: "$CIRRUS_WORKING_DIR/bin;$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin;$PATH" - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work - pub_cache: - folder: $APPDATA\Pub\Cache - fingerprint_script: - - ps: $Env:OS; Get-ChildItem -Path "$Env:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch - flutter_pkg_cache: - folder: bin\cache\pkg - fingerprint_script: echo %OS% & type bin\internal\*.version - artifacts_cache: - folder: bin\cache\artifacts - fingerprint_script: echo %OS% & type bin\internal\engine.version - setup_script: - - flutter config --no-analytics - - flutter doctor -v - - flutter update-packages - - git fetch origin master - test_all_script: - - dart --enable-asserts dev\bots\test.dart - matrix: - # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests_general-windows - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: general - # all of the tests in test/commands/create_test - - name: tool_tests_commands-windows - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: commands - # all of the tests in test/integration for packages/flutter_tools - - name: tool_tests_integration-windows - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: integration + - name: firebase_test_lab_tests-linux # linux-only + environment: + # Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of + # October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM; + # doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it + # on the high end of the 20-25 minute range. (This makes sense, as it's just driving the + # Firebase test lab remotely.) Less than 4G of RAM made it go OOM. + CLOUDSDK_CORE_DISABLE_PROMPTS: 1 + GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec] + script: + - ./dev/bots/firebase_testlab.sh + - name: deploy_gallery-linux # linux- and macos- only + depends_on: + - analyze-linux + - framework_tests-widgets-linux + - framework_tests-libraries-linux + - framework_tests-misc-linux + - tool_tests-general-linux + - tool_tests-commands-linux + - tool_tests-integration-linux + - build_tests-linux + - hostonly_devicelab_tests-0-linux + - hostonly_devicelab_tests-1-linux + - hostonly_devicelab_tests-2-linux + - hostonly_devicelab_tests-3-linux + - hostonly_devicelab_tests-4-linux + - hostonly_devicelab_tests-5_last-linux + - firebase_test_lab_tests-linux + environment: + # As of October 2019, 1 CPU and 4G of RAM let deploy_gallery-linux finish in about 15 + # minutes, once it got started. + GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346] + ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0f2aca35f05b26add5d9edea2a7449341269a2b7e22d5c667f876996e2e8bc44ff1369431ebf73b7c5581fd95d0e5902] + script: + - ./dev/bots/deploy_gallery.sh + +# WINDOWS SHARDS task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' windows_container: image: cirrusci/android-sdk:28-windowsservercore-2019 os_version: 2019 - cpu: 4 - env: - CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk" + cpu: $CPU + memory: $MEMORY + environment: + # It's not clear that changing the cpu/memory fields here has any impact, so we use + # small numbers... + CPU: 2 # 1-8 without compute credits, 1-30 with + MEMORY: 2G # 256M-24G without compute credits, 256M-90G with + CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\$FLUTTER_SDK_PATH_WITH_SPACE" PATH: "$CIRRUS_WORKING_DIR/bin;$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin;$PATH" - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work pub_cache: folder: $APPDATA\Pub\Cache fingerprint_script: - - ps: $Env:OS; Get-ChildItem -Path "$Env:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch + - ps: $Environment:OS; Get-ChildItem -Path "$Environment:CIRRUS_WORKING_DIR" pubspec.yaml -Recurse | Select-String -Pattern "PUBSPEC CHECKSUM" -SimpleMatch flutter_pkg_cache: folder: bin\cache\pkg fingerprint_script: echo %OS% & type bin\internal\*.version @@ -368,290 +348,285 @@ folder: bin\cache\artifacts fingerprint_script: echo %OS% & type bin\internal\*.version setup_script: + - git clean -xffd + - git fetch origin + - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works. - flutter config --no-analytics - flutter doctor -v - flutter update-packages - git fetch origin master matrix: - - name: bots_tests-windows - skip: "!changesInclude('dev/bots/**')" - test_script: - - cd dev\bots - - pub run test - - name: tests_widgets-windows - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: widgets + - name: framework_tests-widgets-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: GOLDCTL: "C:\\Windows\\Temp\\depot_tools\\goldctl.exe" GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: powershell dev\bots\download_goldctl.ps1 - test_all_script: + script: + - powershell dev\bots\download_goldctl.ps1 - dart --enable-asserts dev\bots\test.dart - - name: tests_framework_other-windows - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: framework_other + + - name: framework_tests-libraries-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: GOLDCTL: "C:\\Windows\\Temp\\depot_tools\\goldctl.exe" GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: powershell dev\bots\download_goldctl.ps1 - test_all_script: + script: + - powershell dev\bots\download_goldctl.ps1 - dart --enable-asserts dev\bots\test.dart - - name: tests_extras-windows - skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: extras - test_all_script: + + - name: framework_tests-misc-windows + # this includes the tests for directories in dev/ + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: - dart --enable-asserts dev\bots\test.dart + + - name: tool_tests-general-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-commands-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-integration-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - dart --enable-asserts ./dev/bots/test.dart + + # TODO(ianh): Enable Web tests on Windows + - name: build_tests-windows - env: - SHARD: build_tests - container: - cpu: 4 - memory: 12G - test_all_script: + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: - dart --enable-asserts dev\bots\test.dart - - name: integration_tests-windows - env: - SHARD: integration_tests - container: - cpu: 4 - memory: 12G - test_all_script: - - dart --enable-asserts dev\bots\test.dart + + - name: hostonly_devicelab_tests-0-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-1-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-2-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-3-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-4-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-5_last-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + script: + - dart --enable-asserts ./dev/bots/test.dart + + # TODO(ianh): name: add_to_app_tests-windows + - name: customer_testing-windows - test_script: + script: - CMD /S /C "IF EXIST "bin\cache\pkg\tests\" RMDIR /S /Q bin\cache\pkg\tests" - git clone https://github.com/flutter/tests.git bin\cache\pkg\tests - dart --enable-asserts dev\customer_testing\run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test - - name: gradle_tests-windows-shard-1 - env: - SHARD: integration_tests - SUBSHARD: gradle1 - test_script: - - dart --enable-asserts dev\bots\test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_tests-windows-shard-2 - env: - SHARD: integration_tests - SUBSHARD: gradle2 - test_script: - - dart --enable-asserts dev\bots\test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_embedding_v2_tests-windows-shard-1 - env: - SHARD: integration_tests - SUBSHARD: gradle1 - ENABLE_ANDROID_EMBEDDING_V2: 'true' - test_script: - - dart --enable-asserts dev\bots\test.dart - container: - cpu: 4 - memory: 12G - - name: gradle_embedding_v2_tests-windows-shard-2 - env: - SHARD: integration_tests - SUBSHARD: gradle2 - ENABLE_ANDROID_EMBEDDING_V2: 'true' - test_script: - - dart --enable-asserts dev\bots\test.dart - container: - cpu: 4 - memory: 12G +# MACOS SHARDS +# Mac doesn't use caches because they apparently take longer to populate and save +# than just fetching the data in the first place. task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - name: deploy_gallery-macos - depends_on: - - analyze - env: - # Name the SDK directory to include a space so that we constantly - # test path names with spaces in them. - CIRRUS_WORKING_DIR: "/tmp/flutter sdk" - SHARD: deploy_gallery - # Apple Fastlane password. - FASTLANE_PASSWORD: ENCRYPTED[4b1f0b8d52874e9de965acd46c79743f3b81f3a513614179b9be7cf53dc8258753e257bdadb11a298ee455259df21865] - # Private repo for publishing certificates. - PUBLISHING_MATCH_CERTIFICATE_REPO: ENCRYPTED[3c0e78877d933fc80107aa6f3790fd1cf927250b852d6cb53202be696b4903ed8ca839b809626aaf18050bf7e436fab7] - PUBLISHING_MATCH_REPO_TOKEN: ENCRYPTED[3d1230b744c6ed6c788a91bec741b769401dbcd426b18f9af8080bfeefdfc21913ca4047980c5b5b7ce823f12e7b6b19] - # Apple Certificates Match Passphrase - MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73] osx_instance: - image: mojave-xcode-10.2 - # occasionally the clock on these machines is out of sync - # with the actual time - this should help to verify - print_date_script: - - date - install_cocoapods_script: - - sudo gem install cocoapods --no-document - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD - setup_script: - - bin/flutter config --no-analytics - - bin/flutter update-packages - test_all_script: - - ./dev/bots/deploy_gallery.sh - -task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - osx_instance: - image: mojave-xcode-10.2 - depends_on: - - analyze - env: - CIRRUS_WORKING_DIR: "/tmp/flutter sdk" - COCOAPODS_DISABLE_STATS: true - print_date_script: - - date - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work - setup_script: - - bin/flutter config --no-analytics - - bin/flutter doctor -v - - bin/flutter update-packages - test_all_script: - - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 - - bin/cache/dart-sdk/bin/dart --enable-asserts dev/bots/test.dart - matrix: - # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests_general-macos - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: general - # all of the tests in test/commands/create_test - - name: tool_tests_commands-macos - skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: commands - # all of the tests in test/integration for packages/flutter_tools - - name: tool_tests_integration-macos - only_if: $CIRRUS_BRANCH == 'master' - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tool_tests - SUBSHARD: integration - -task: - use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' - osx_instance: - image: mojave-xcode-10.2 - depends_on: - - analyze - env: - CIRRUS_WORKING_DIR: "/tmp/flutter sdk" - COCOAPODS_DISABLE_STATS: true + image: mojave-xcode-10.2-flutter # see https://cirrus-ci.org/guide/macOS/ for list of images (we should update regularly) + # cpu is always 2 + # memory is always 8G + environment: + CIRRUS_WORKING_DIR: "/tmp/$FLUTTER_SDK_PATH_WITH_SPACE" + FLUTTER_FRAMEWORK_DIR: "$CIRRUS_WORKING_DIR/bin/cache/artifacts/engine/ios/" PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH" - # occasionally the clock on these machines is out of sync - # with the actual time - this should help to verify - print_date_script: - - date - install_cocoapods_script: - - sudo gem install cocoapods --no-document - git_fetch_script: - - git clean -xfd - - git fetch origin - - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work + COCOAPODS_DISABLE_STATS: true + CPU: 2 + MEMORY: 8G setup_script: - - bin/flutter config --no-analytics - - bin/flutter doctor -v - - bin/flutter update-packages + - date + - which flutter + - sudo gem install cocoapods + - sudo gem install xcpretty + - git clean -xffd + - git fetch origin + - git fetch origin master # To set FETCH_HEAD, so that "git merge-base" works. + - flutter config --no-analytics + - flutter doctor -v + - flutter update-packages + - date + - which flutter + on_failure: + failure_script: + - date + - which flutter matrix: - - name: bots_tests-macos - skip: "!changesInclude('dev/bots/**')" - test_script: - - (cd ./dev/bots && pub run test) - - name: tests_widgets-macos - only_if: $CIRRUS_BRANCH == 'master' - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: widgets + - name: framework_tests-widgets-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: ./dev/bots/download_goldctl.sh - test_all_script: + script: - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - ./dev/bots/download_goldctl.sh - dart --enable-asserts dev/bots/test.dart - on_failure: - print_failure_time_script: date - - name: tests_framework_other-macos - only_if: $CIRRUS_BRANCH == 'master' - env: - GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] - SHARD: tests - SUBSHARD: framework_other + + - name: framework_tests-libraries-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + environment: GOLDCTL: "$CIRRUS_WORKING_DIR/depot_tools/goldctl" GOLD_SERVICE_ACCOUNT: ENCRYPTED[3afeea5ac7201151c3d0dc9648862f0462b5e4f55dc600ca8b692319622f7c3eda3d577b1b16cc2ef0311b7314c1c095] - goldctl_script: ./dev/bots/download_goldctl.sh - test_all_script: + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - ./dev/bots/download_goldctl.sh + - dart --enable-asserts dev/bots/test.dart + + - name: framework_tests-misc-macos + # this includes the tests for directories in dev/ + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 - dart --enable-asserts dev/bots/test.dart - on_failure: - print_failure_time_script: date - - name: integration_tests-macos - only_if: $CIRRUS_BRANCH == 'master' - env: - SHARD: integration_tests - test_all_script: + + - name: tool_tests-general-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-commands-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: tool_tests-integration-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter_tools/**', 'bin/internal/**') || $CIRRUS_PR == ''" + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + # TODO(ianh): Enable Web tests on macOS. + + - name: build_tests-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-0-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-1-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-2-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-3-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-4-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: hostonly_devicelab_tests-5_last-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - dart --enable-asserts ./dev/bots/test.dart + + - name: add_to_app_tests-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + skip: true # https://github.com/flutter/flutter/pull/42444 + script: - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 - dart --enable-asserts dev/bots/test.dart - - name: add2app-macos - skip: true # https://github.com/flutter/flutter/issues/39507 - env: - SHARD: add2app_test - setup_xcpretty_script: - - sudo gem install xcpretty - test_all_script: - - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 - - dart --enable-asserts dev/bots/test.dart + - name: customer_testing-macos - test_script: + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 - rm -rf bin/cache/pkg/tests - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test + - name: deploy_gallery-macos # linux- and macos- only + depends_on: + - analyze-linux + - framework_tests-widgets-macos + - framework_tests-libraries-macos + - framework_tests-misc-macos + - tool_tests-general-macos + - tool_tests-commands-macos + - tool_tests-integration-macos + - build_tests-macos + - hostonly_devicelab_tests-0-macos + - hostonly_devicelab_tests-1-macos + - hostonly_devicelab_tests-2-macos + - hostonly_devicelab_tests-3-macos + - hostonly_devicelab_tests-4-macos + - hostonly_devicelab_tests-5_last-macos + - firebase_test_lab_tests-linux + environment: + # Apple Fastlane password. + FASTLANE_PASSWORD: ENCRYPTED[4b1f0b8d52874e9de965acd46c79743f3b81f3a513614179b9be7cf53dc8258753e257bdadb11a298ee455259df21865] + # Private repo for publishing certificates. + PUBLISHING_MATCH_CERTIFICATE_REPO: ENCRYPTED[3c0e78877d933fc80107aa6f3790fd1cf927250b852d6cb53202be696b4903ed8ca839b809626aaf18050bf7e436fab7] + PUBLISHING_MATCH_REPO_TOKEN: ENCRYPTED[3d1230b744c6ed6c788a91bec741b769401dbcd426b18f9af8080bfeefdfc21913ca4047980c5b5b7ce823f12e7b6b19] + # Apple Certificates Match Passphrase + MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73] + script: + - ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976 + - ./dev/bots/deploy_gallery.sh + docker_builder: # Only build a new docker image when we tag a release (for dev, beta, or release.) only_if: $CIRRUS_TAG != '' - env: + environment: GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6] depends_on: - - docs - - analyze - - tests_widgets-linux - - tests_framework_other-linux - - tests_extras-linux - - tool_tests_general-linux - - tool_tests_commands-linux - - tool_tests_integration-linux + - docs-linux + - analyze-linux + - framework_tests-widgets-linux + - framework_tests-libraries-linux + - framework_tests-misc-linux + - tool_tests-general-linux + - tool_tests-commands-linux + - tool_tests-integration-linux - build_tests-linux - - integration_tests-linux - - gradle_tests-linux-shard-1 - - gradle_tests-linux-shard-2 - - gradle_embedding_v2_tests-linux-shard-1 - - gradle_embedding_v2_tests-linux-shard-2 - - gradle_tests-windows-shard-1 - - gradle_tests-windows-shard-2 - - gradle_embedding_v2_tests-windows-shard-1 - - gradle_embedding_v2_tests-windows-shard-2 - - release_smoke_tests - build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh" - login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh" - push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh" + - hostonly_devicelab_tests-0-linux + - hostonly_devicelab_tests-1-linux + - hostonly_devicelab_tests-2-linux + - hostonly_devicelab_tests-3-linux + - hostonly_devicelab_tests-4-linux + - hostonly_devicelab_tests-5_last-linux + - firebase_test_lab_tests-linux + script: + - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh" + - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh" + - "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh"
diff --git a/dev/README.md b/dev/README.md index afe8f26..3c287dd 100644 --- a/dev/README.md +++ b/dev/README.md
@@ -2,3 +2,6 @@ during development of the framework. The tools in this directory should not be necessary for developing Flutter applications, though of course they may be interesting if you are curious. + +The tests in this directory are run in the `framework_tests_misc-*` +shards.
diff --git a/dev/bots/accept_android_sdk_licenses.sh b/dev/bots/accept_android_sdk_licenses.sh new file mode 100755 index 0000000..a5b6abb --- /dev/null +++ b/dev/bots/accept_android_sdk_licenses.sh
@@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +# This script is only meant to be run by the Cirrus CI system, not locally. +# It must be run from the root of the Flutter repo. + +function error() { + echo "$@" 1>&2 +} + +function accept_android_licenses() { + yes "y" | flutter doctor --android-licenses > /dev/null 2>&1 +} + +echo "Flutter SDK directory is: $PWD" + +# Accept licenses. +echo "Accepting Android licenses." +accept_android_licenses || (error "Accepting Android licenses failed." && false)
diff --git a/dev/bots/cirrus_setup.sh b/dev/bots/cirrus_setup.sh deleted file mode 100755 index 451bcbc..0000000 --- a/dev/bots/cirrus_setup.sh +++ /dev/null
@@ -1,31 +0,0 @@ -#!/bin/bash -set -e - -function error() { - echo "$@" 1>&2 -} - -# This script is only meant to be run by the Cirrus CI system, not locally. -# It must be run from the root of the Flutter repo. - -function accept_android_licenses() { - yes "y" | flutter doctor --android-licenses > /dev/null 2>&1 -} - -echo "Flutter SDK directory is: $PWD" - -# Run flutter to download dependencies and precompile things, and to disable -# analytics on the bots. -echo "Downloading build dependencies and pre-compiling Flutter snapshot" -./bin/flutter config --no-analytics - -# Run doctor, to print it to the log for debugging purposes. -./bin/flutter doctor -v - -# Accept licenses. -echo "Accepting Android licenses." -accept_android_licenses || (error "Accepting Android licenses failed." && false) - -# Run pub get in all the repo packages. -echo "Updating packages for Flutter." -./bin/flutter update-packages
diff --git a/dev/bots/deploy_gallery.sh b/dev/bots/deploy_gallery.sh index f372fec..86d34d9 100755 --- a/dev/bots/deploy_gallery.sh +++ b/dev/bots/deploy_gallery.sh
@@ -24,88 +24,90 @@ cd "$FLUTTER_ROOT" -if [[ "$SHARD" = "deploy_gallery" ]]; then - version="$(<version)" - if [[ "$OS" == "linux" ]]; then - echo "Building Flutter Gallery $version for Android..." - - # ANDROID_SDK_ROOT must be set in the env. +version="$(<version)" +if [[ "$OS" == "linux" ]]; then + echo "Building Flutter Gallery $version for Android..." + # ANDROID_SDK_ROOT must be set in the env. + ( + cd examples/flutter_gallery + flutter build apk --release -t lib/main_publish.dart + ) + echo "Android Flutter Gallery built" + if [[ -z "$CIRRUS_PR" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then + echo "Deploying Flutter Gallery $version to Play Store..." + set +x # Don't echo back the below. + if [ -n "$ANDROID_GALLERY_UPLOAD_KEY" ]; then + echo "$ANDROID_GALLERY_UPLOAD_KEY" | base64 --decode > /root/.android/debug.keystore + fi + set -x ( - cd examples/flutter_gallery - flutter build apk --release -t lib/main_publish.dart + cd examples/flutter_gallery/android + fastlane deploy_play_store ) - echo "Android Flutter Gallery built" - if [[ -z "$CIRRUS_PR" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then - echo "Deploying Flutter Gallery $version to Play Store..." - set +x # Don't echo back the below. - if [ -n "$ANDROID_GALLERY_UPLOAD_KEY" ]; then - echo "$ANDROID_GALLERY_UPLOAD_KEY" | base64 --decode > /root/.android/debug.keystore - fi - set -x + else + echo "(Not deploying; Flutter Gallery is only deployed to Play store for tagged dev branch commits.)" + fi +elif [[ "$OS" == "darwin" ]]; then + echo "Building Flutter Gallery $version for iOS..." + ( + cd examples/flutter_gallery + flutter build ios --release --no-codesign -t lib/main_publish.dart + + # flutter build ios will run CocoaPods script. Check generated locations. + if [[ ! -d "ios/Pods" ]]; then + echo "Error: pod install failed to setup plugins" + exit 1 + fi + + if [[ ! -d "ios/.symlinks/plugins" ]]; then + echo "Error: pod install failed to setup plugin symlinks" + exit 1 + fi + + if [[ -d "ios/.symlinks/flutter" ]]; then + echo "Error: pod install created flutter symlink" + exit 1 + fi + + if [[ ! -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets" ]]; then + echo "Error: flutter_assets not assembled" + exit 1 + fi + + if [[ + -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/isolate_snapshot_data" || + -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/kernel_blob.bin" || + -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/vm_snapshot_data" + ]]; then + echo "Error: compiled debug version of app with --release flag" + exit 1 + fi + ) + echo "iOS Flutter Gallery built" + if [[ -z "$CIRRUS_PR" ]]; then + if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then + echo "Archiving with distribution profile and deploying to TestFlight..." ( - cd examples/flutter_gallery/android - fastlane deploy_play_store + cd examples/flutter_gallery/ios + export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV" + fastlane build_and_deploy_testflight upload:true ) else - echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits" + # On iOS the signing can break as well, so we verify this regularly (not just + # on tagged dev branch commits). We can only do this post-merge, though, because + # the secrets aren't available on PRs. + echo "Testing archiving with distribution profile..." + ( + cd examples/flutter_gallery/ios + fastlane build_and_deploy_testflight + ) + echo "(Not deploying; Flutter Gallery is only deployed to TestFlight for tagged dev branch commits.)" fi - elif [[ "$OS" == "darwin" ]]; then - echo "Building Flutter Gallery $version for iOS..." - ( - cd examples/flutter_gallery - flutter build ios --release --no-codesign -t lib/main_publish.dart - - # flutter build ios will run CocoaPods script. Check generated locations. - if [[ ! -d "ios/Pods" ]]; then - echo "Error: pod install failed to setup plugins" - exit 1 - fi - - if [[ ! -d "ios/.symlinks/plugins" ]]; then - echo "Error: pod install failed to setup plugin symlinks" - exit 1 - fi - - if [[ -d "ios/.symlinks/flutter" ]]; then - echo "Error: pod install created flutter symlink" - exit 1 - fi - - if [[ ! -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets" ]]; then - echo "Error: flutter_assets not assembled" - exit 1 - fi - - if [[ - -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/isolate_snapshot_data" || - -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/kernel_blob.bin" || - -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/vm_snapshot_data" - ]]; then - echo "Error: compiled debug version of app with --release flag" - exit 1 - fi - ) - echo "iOS Flutter Gallery built" - if [[ -z "$CIRRUS_PR" ]]; then - if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then - echo "Archiving with distribution profile and deploying to TestFlight..." - ( - cd examples/flutter_gallery/ios - export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV" - fastlane build_and_deploy_testflight upload:true - ) - else - echo "Archiving with distribution profile..." - ( - cd examples/flutter_gallery/ios - fastlane build_and_deploy_testflight - ) - echo "Archive is only deployed to TestFlight on tagged dev branch commits" - fi - else - echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits" - fi + else + echo "(Not archiving or deploying; Flutter Gallery archiving is only tested post-commit.)" fi else - echo "Doing nothing: not on the 'deploy_gallery' SHARD." + echo "Unknown OS: $OS" + echo "Aborted." + exit 1 fi
diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index 634a232..f240a5c 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart
@@ -95,7 +95,6 @@ OutputMode outputMode = OutputMode.print, CapturedOutput output, bool skip = false, - bool expectFlaky = false, bool Function(String) removeLine, }) async { assert((outputMode == OutputMode.capture) == (output != null), @@ -145,10 +144,6 @@ output.stderr = _flattenToString(await savedStderr); } - // If the test is flaky we don't care about the actual exit. - if (expectFlaky) - return; - if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) { if (failureMessage != null) { print(failureMessage);
diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 9914dd2..fea54fb 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart
@@ -4,11 +4,11 @@ import 'dart:async'; import 'dart:io'; +import 'dart:math' as math; import 'package:googleapis/bigquery/v2.dart' as bq; import 'package:googleapis_auth/auth_io.dart' as auth; import 'package:http/http.dart' as http; -import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; import 'flutter_compact_formatter.dart'; @@ -30,21 +30,54 @@ final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub'); final String pubCache = path.join(flutterRoot, '.pub-cache'); final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools'); + +/// The arguments to pass to `flutter test` (typically the local engine +/// configuration) -- prefilled with the arguments passed to test.dart. final List<String> flutterTestArgs = <String>[]; final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true'; final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true'; -const Map<String, ShardRunner> _kShards = <String, ShardRunner>{ - 'tests': _runTests, - 'web_tests': _runWebTests, - 'tool_tests': _runToolTests, - 'tool_coverage': _runToolCoverage, - 'build_tests': _runBuildTests, - 'coverage': _runCoverage, - 'integration_tests': _runIntegrationTests, - 'add2app_test': _runAdd2AppTest, -}; +/// The number of Cirrus jobs that run host-only devicelab tests in parallel. +/// +/// WARNING: if you change this number, also change .cirrus.yml +/// and make sure it runs _all_ shards. +const int kDeviceLabShardCount = 6; + +/// The number of Cirrus jobs that run Web tests in parallel. +/// +/// WARNING: if you change this number, also change .cirrus.yml +/// and make sure it runs _all_ shards. +/// +/// The last shard also runs the Web plugin tests. +const int kWebShardCount = 6; + +/// Maximum number of Web tests to run in a single `flutter test`. We found that +/// large batches can get flaky, possibly because we reuse a single instance of +/// the browser, and after many tests the browser's state gets corrupted. +const int kWebBatchSize = 20; + +/// Tests that we don't run on Web for various reasons. +// +// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60 +const List<String> kWebTestDirectoryBlacklist = <String>[ + 'cupertino', + 'examples', + 'material', +]; +const List<String> kWebTestFileBlacklist = <String>[ + 'test/widgets/heroes_test.dart', + 'test/widgets/text_test.dart', + 'test/widgets/selectable_text_test.dart', + 'test/widgets/color_filter_test.dart', + 'test/widgets/editable_text_cursor_test.dart', + 'test/widgets/shadow_test.dart', + 'test/widgets/raw_keyboard_listener_test.dart', + 'test/widgets/editable_text_test.dart', + 'test/widgets/widget_inspector_test.dart', + 'test/widgets/draggable_test.dart', + 'test/widgets/shortcuts_test.dart', +]; /// When you call this, you can pass additional arguments to pass custom /// arguments to flutter test. For example, you might want to call this @@ -53,31 +86,30 @@ /// /// To run the tool_tests part, run it with SHARD=tool_tests /// -/// For example: +/// Examples: /// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt Future<void> main(List<String> args) async { flutterTestArgs.addAll(args); - - final String shard = Platform.environment['SHARD']; - if (shard != null) { - if (!_kShards.containsKey(shard)) { - print('Invalid shard: $shard'); - print('The available shards are: ${_kShards.keys.join(", ")}'); - exit(1); - } - print('${bold}SHARD=$shard$reset'); - await _kShards[shard](); - } else { - for (String currentShard in _kShards.keys) { - print('${bold}SHARD=$currentShard$reset'); - await _kShards[currentShard](); - print(''); - } - } + if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) + print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}'); + print('═' * 80); + await _runSmokeTests(); + print('═' * 80); + await selectShard(const <String, ShardRunner>{ + 'add_to_app_tests': _runAddToAppTests, + 'build_tests': _runBuildTests, + 'framework_coverage': _runFrameworkCoverage, + 'framework_tests': _runFrameworkTests, + 'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests, + 'tool_coverage': _runToolCoverage, + 'tool_tests': _runToolTests, + 'web_tests': _runWebTests, + }); } Future<void> _runSmokeTests() async { + print('${green}Running smoketests...$reset'); // Verify that the tests actually return failure on failure and success on // success. final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); @@ -108,10 +140,11 @@ script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), expectFailure: true, printOutput: false, - outputChecker: (CapturedOutput output) => - output.stdout.contains('failingPendingTimerTest') - ? null - : 'Failed to find the stack trace for the pending Timer.', + outputChecker: (CapturedOutput output) { + return output.stdout.contains('failingPendingTimerTest') + ? null + : 'Failed to find the stack trace for the pending Timer.'; + } ); // We run the remaining smoketests in parallel, because they each take some // time to run (e.g. compiling), so we don't want to run them in series, @@ -153,8 +186,11 @@ ); // Verify that we correctly generated the version file. - final bool validVersion = await verifyVersion(path.join(flutterRoot, 'version')); - if (!validVersion) { + final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version'))); + if (versionError != null) { + print(redLine); + print(versionError); + print(redLine); exit(1); } } @@ -179,7 +215,7 @@ final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes); return bq.BigqueryApi(client); } catch (e) { - print('Failed to get BigQuery API client.'); + print('${red}Failed to get BigQuery API client.$reset'); print(e); return null; } @@ -203,7 +239,6 @@ Future<void> _runToolTests() async { final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); - await _runSmokeTests(); const String kDotShard = '.shard'; const String kTest = 'test'; @@ -232,11 +267,10 @@ await selectSubshard(subshards); } -/// Verifies that AOT, APK, and IPA (if on macOS) builds the -/// examples apps without crashing. It does not actually -/// launch the apps. That happens later in the devicelab. This is -/// just a smoke-test. In particular, this will verify we can build -/// when there are spaces in the path name for the Flutter SDK and +/// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps +/// without crashing. It does not actually launch the apps. That happens later +/// in the devicelab. This is just a smoke-test. In particular, this will verify +/// we can build when there are spaces in the path name for the Flutter SDK and /// target app. Future<void> _runBuildTests() async { final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list(); @@ -245,10 +279,11 @@ continue; } final String examplePath = fileEntity.path; - await _flutterBuildAot(examplePath); await _flutterBuildApk(examplePath); - await _flutterBuildIpa(examplePath); + if (Platform.isMacOS) { + await _flutterBuildIpa(examplePath); + } } // Web compilation tests. await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart')); @@ -256,12 +291,44 @@ await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'), path.join('lib', 'dart_io_import.dart'), ); +} - print('${bold}DONE: All build tests successful.$reset'); +Future<void> _flutterBuildAot(String relativePathToApplication) async { + print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...'); + await runCommand(flutter, + <String>['build', 'aot', '-v'], + workingDirectory: path.join(flutterRoot, relativePathToApplication), + ); +} + +Future<void> _flutterBuildApk(String relativePathToApplication) async { + print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...'); + await runCommand(flutter, + <String>['build', 'apk', '--debug', '-v'], + workingDirectory: path.join(flutterRoot, relativePathToApplication), + ); +} + +Future<void> _flutterBuildIpa(String relativePathToApplication) async { + assert(Platform.isMacOS); + print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...'); + // Install Cocoapods. We don't have these checked in for the examples, + // and build ios doesn't take care of it automatically. + final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile')); + if (podfile.existsSync()) { + await runCommand('pod', + <String>['install'], + workingDirectory: podfile.parent.path, + ); + } + await runCommand(flutter, + <String>['build', 'ios', '--no-codesign', '--debug', '-v'], + workingDirectory: path.join(flutterRoot, relativePathToApplication), + ); } Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async { - print('Running Dart2JS build tests...'); + print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...'); await runCommand(flutter, <String>['build', 'web', '-v', '--target=$target'], workingDirectory: path.join(flutterRoot, relativePathToApplication), @@ -270,211 +337,101 @@ 'FLUTTER_WEB': 'true', }, ); - print('Done.'); } -Future<void> _flutterBuildAot(String relativePathToApplication) async { - print('Running AOT build tests...'); - await runCommand(flutter, - <String>['build', 'aot', '-v'], - workingDirectory: path.join(flutterRoot, relativePathToApplication), - expectNonZeroExit: false, - ); - print('Done.'); -} - -Future<void> _flutterBuildApk(String relativePathToApplication) async { - if ( - (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) && - (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) { - return; - } - print('Running APK build tests...'); - await runCommand(flutter, - <String>['build', 'apk', '--debug', '-v'], - workingDirectory: path.join(flutterRoot, relativePathToApplication), - expectNonZeroExit: false, - ); - print('Done.'); -} - -Future<void> _flutterBuildIpa(String relativePathToApplication) async { - if (!Platform.isMacOS) { - return; - } - print('Running IPA build tests...'); - // Install Cocoapods. We don't have these checked in for the examples, - // and build ios doesn't take care of it automatically. - final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile')); - if (podfile.existsSync()) { - await runCommand('pod', - <String>['install'], - workingDirectory: podfile.parent.path, - expectNonZeroExit: false, +Future<void> _runAddToAppTests() async { + if (Platform.isMacOS) { + print('${green}Running add-to-app iOS integration tests$reset...'); + final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app'); + await runCommand('./build_and_test.sh', + <String>[], + workingDirectory: addToAppDir, ); } - await runCommand(flutter, - <String>['build', 'ios', '--no-codesign', '--debug', '-v'], - workingDirectory: path.join(flutterRoot, relativePathToApplication), - expectNonZeroExit: false, - ); - print('Done.'); } -Future<void> _runAdd2AppTest() async { - if (!Platform.isMacOS) { - return; - } - print('Running Add2App iOS integration tests...'); - final String add2AppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app'); - await runCommand('./build_and_test.sh', - <String>[], - workingDirectory: add2AppDir, - expectNonZeroExit: false, - ); - print('Done.'); -} - -Future<void> _runTests() async { +Future<void> _runFrameworkTests() async { final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); - await _runSmokeTests(); - final String subShard = Platform.environment['SUBSHARD']; Future<void> runWidgets() async { - await _runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - tableData: bigqueryApi?.tabledata, - tests: <String>[ - path.join('test', 'widgets') + path.separator, - ], - ); - // Only packages/flutter/test/widgets/widget_inspector_test.dart really - // needs to be run with --track-widget-creation but it is nice to run - // all of the tests in package:flutter with the flag to ensure that - // the Dart kernel transformer triggered by the flag does not break anything. + print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset'); await _runFlutterTest( path.join(flutterRoot, 'packages', 'flutter'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata, - tests: <String>[ - path.join('test', 'widgets') + path.separator, - ], + tests: <String>[ path.join('test', 'widgets') + path.separator ], ); + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: <String>['--no-track-widget-creation'], + tableData: bigqueryApi?.tabledata, + tests: <String>[ path.join('test', 'widgets') + path.separator ], + ); + // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation + await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata); } - Future<void> runFrameworkOthers() async { + Future<void> runLibraries() async { final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test')) .listSync(followLinks: false, recursive: false) .whereType<Directory>() .where((Directory dir) => dir.path.endsWith('widgets') == false) - .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) + .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) .toList(); - - print('Running tests for: ${tests.join(';')}'); - - await _runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - tableData: bigqueryApi?.tabledata, - tests: tests, - ); - // Only packages/flutter/test/widgets/widget_inspector_test.dart really - // needs to be run with --track-widget-creation but it is nice to run - // all of the tests in package:flutter with the flag to ensure that - // the Dart kernel transformer triggered by the flag does not break anything. + print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset'); await _runFlutterTest( path.join(flutterRoot, 'packages', 'flutter'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata, tests: tests, ); + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: <String>['--no-track-widget-creation'], + tableData: bigqueryApi?.tabledata, + tests: tests, + ); } - Future<void> runExtras() async { - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata); + Future<void> runMisc() async { + print('${green}Running package tests$reset for directories other than packages/flutter'); await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata); await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata); await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata); - // Regression test to ensure that code outside of package:flutter can run - // with --track-widget-creation. - await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata); - // Smoke test for code generation. - await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: <String, String>{ - 'FLUTTER_EXPERIMENTAL_BUILD': 'true', - }); - } - switch (subShard) { - case 'widgets': - await runWidgets(); - break; - case 'framework_other': - await runFrameworkOthers(); - break; - case 'extras': - runExtras(); - break; - default: - print('Unknown sub-shard $subShard, running all tests!'); - await runWidgets(); - await runFrameworkOthers(); - await runExtras(); - + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest( + path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), + tableData: bigqueryApi?.tabledata, + environment: <String, String>{ + 'FLUTTER_EXPERIMENTAL_BUILD': 'true', + }, + ); } - print('${bold}DONE: All tests successful.$reset'); + await selectSubshard(<String, ShardRunner>{ + 'widgets': runWidgets, + 'libraries': runLibraries, + 'misc': runMisc, + }); } -// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60 -const List<String> kWebTestDirectoryBlacklist = <String>[ - 'test/cupertino', - 'test/examples', - 'test/material', -]; -const List<String> kWebTestFileBlacklist = <String>[ - 'test/widgets/heroes_test.dart', - 'test/widgets/text_test.dart', - 'test/widgets/selectable_text_test.dart', - 'test/widgets/color_filter_test.dart', - 'test/widgets/editable_text_cursor_test.dart', - 'test/widgets/shadow_test.dart', - 'test/widgets/raw_keyboard_listener_test.dart', - 'test/widgets/editable_text_test.dart', - 'test/widgets/widget_inspector_test.dart', - 'test/widgets/draggable_test.dart', - 'test/widgets/shortcuts_test.dart', -]; - -Future<void> _runWebTests() async { - final Directory flutterPackageDir = Directory(path.join(flutterRoot, 'packages', 'flutter')); - final Directory testDir = Directory(path.join(flutterPackageDir.path, 'test')); - - final List<String> directories = testDir - .listSync() - .whereType<Directory>() - .map<String>((Directory dir) => path.relative(dir.path, from: flutterPackageDir.path)) - .where((String relativePath) => !kWebTestDirectoryBlacklist.contains(relativePath)) - .toList(); - - await _runFlutterWebTest(flutterPackageDir.path, tests: directories); - await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter_web_plugins'), tests: <String>['test']); -} - -Future<void> _runCoverage() async { +Future<void> _runFrameworkCoverage() async { final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info')); if (!coverageFile.existsSync()) { print('${red}Coverage file not found.$reset'); - print('Expected to find: ${coverageFile.absolute}'); - print('This file is normally obtained by running `flutter update-packages`.'); + print('Expected to find: $cyan${coverageFile.absolute}$reset'); + print('This file is normally obtained by running `${green}flutter update-packages$reset`.'); exit(1); } coverageFile.deleteSync(); @@ -483,11 +440,98 @@ ); if (!coverageFile.existsSync()) { print('${red}Coverage file not found.$reset'); - print('Expected to find: ${coverageFile.absolute}'); - print('This file should have been generated by the `flutter test --coverage` script, but was not.'); + print('Expected to find: $cyan${coverageFile.absolute}$reset'); + print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.'); exit(1); } - print('${bold}DONE: Coverage collection successful.$reset'); +} + +Future<void> _runWebTests() async { + final Map<String, ShardRunner> subshards = <String, ShardRunner>{}; + + final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter')); + final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test')); + + final List<String> allTests = flutterPackageTestDirectory + .listSync() + .whereType<Directory>() + .where((Directory directory) => !kWebTestDirectoryBlacklist.contains(path.basename(directory.path))) + .expand((Directory directory) => directory + .listSync(recursive: true) + .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart')) + ) + .whereType<File>() + .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path)) + .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)) + .toList() + // Finally we shuffle the list because we want the average cost per file to be uniformly + // distributed. If the list is not sorted then different shards and batches may have + // very different characteristics. + // We use a constant seed for repeatability. + ..shuffle(math.Random(0)); + + assert(kWebShardCount >= 1); + final int testsPerShard = (allTests.length / kWebShardCount).ceil(); + assert(testsPerShard * kWebShardCount >= allTests.length); + + // This for loop computes all but the last shard. + for (int index = 0; index < kWebShardCount - 1; index += 1) { + subshards['$index'] = () => _runFlutterWebTest( + flutterPackageDirectory.path, + allTests.sublist( + index * testsPerShard, + (index + 1) * testsPerShard, + ), + ); + } + + // The last shard also runs the flutter_web_plugins tests. + // + // We make sure the last shard ends in _last so it's easier to catch mismatches + // between `.cirrus.yml` and `test.dart`. + subshards['${kWebShardCount - 1}_last'] = () async { + await _runFlutterWebTest( + flutterPackageDirectory.path, + allTests.sublist( + (kWebShardCount - 1) * testsPerShard, + allTests.length, + ), + ); + await _runFlutterWebTest( + path.join(flutterRoot, 'packages', 'flutter_web_plugins'), + <String>['test'], + ); + }; + + await selectSubshard(subshards); +} + +Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async { + final List<String> batch = <String>[]; + for (int i = 0; i < tests.length; i += 1) { + final String testFilePath = tests[i]; + batch.add(testFilePath); + if (batch.length == kWebBatchSize || i == tests.length - 1) { + await runCommand( + flutter, + <String>[ + 'test', + if (ciProvider == CiProviders.cirrus) + '--concurrency=1', // do not parallelize on Cirrus, to reduce flakiness + '-v', + '--platform=chrome', + ...?flutterTestArgs, + ...batch, + ], + workingDirectory: workingDirectory, + environment: <String, String>{ + 'FLUTTER_WEB': 'true', + 'FLUTTER_LOW_RESOURCE_MODE': 'true', + }, + ); + batch.clear(); + } + } } Future<void> _pubRunTest(String workingDirectory, { @@ -496,14 +540,33 @@ bool useBuildRunner = false, bq.TabledataResourceApi tableData, }) async { - final List<String> args = <String>['run', '--verbose']; + final List<String> args = <String>['run']; if (useBuildRunner) { - args.addAll(<String>['build_runner', 'test', '--']); + final String posixTestPath = path.posix.joinAll(path.split(testPath)); + args.addAll(<String>[ + 'build_runner', + 'test', + '--build-filter=$posixTestPath/*.dill', + '--build-filter=$posixTestPath/**/*.dill', + '--', + ]); } else { args.add('test'); } args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact'); - args.add('-j1'); // TODO(ianh): Scale based on CPUs. + int cpus; + final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml + if (cpuVariable != null) { + cpus = int.tryParse(cpuVariable, radix: 10); + if (cpus == null) { + print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset'); + print('Actual value: "$cpuVariable"'); + exit(1); + } + } else { + cpus = 2; // Don't default to 1, otherwise we won't catch race conditions. + } + args.add('-j$cpus'); if (!hasColor) args.add('--no-color'); if (testPath != null) @@ -547,238 +610,6 @@ } } -void deleteFile(String path) { - // There's a race condition here but in theory we're not racing anyone - // while this script runs, so should be ok. - final File file = File(path); - if (file.existsSync()) - file.deleteSync(); -} - -enum CiProviders { - cirrus, - luci, -} - -CiProviders _getCiProvider() { - if (Platform.environment['CIRRUS_CI'] == 'true') { - return CiProviders.cirrus; - } - if (Platform.environment['LUCI_CONTEXT'] != null) { - return CiProviders.luci; - } - return null; -} - -String _getCiProviderName() { - switch(_getCiProvider()) { - case CiProviders.cirrus: - return 'cirrusci'; - case CiProviders.luci: - return 'luci'; - } - return 'unknown'; -} - -int _getPrNumber() { - switch(_getCiProvider()) { - case CiProviders.cirrus: - return Platform.environment['CIRRUS_PR'] == null - ? -1 - : int.tryParse(Platform.environment['CIRRUS_PR']); - case CiProviders.luci: - return -1; // LUCI doesn't know about this. - } - return -1; -} - -Future<String> _getAuthors() async { - final String exe = Platform.isWindows ? '.exe' : ''; - final String author = await runAndGetStdout( - 'git$exe', <String>['-c', 'log.showSignature=false', 'log', _getGitHash(), '--pretty="%an <%ae>"'], - workingDirectory: flutterRoot, - ).first; - return author; -} - -String _getCiUrl() { - switch(_getCiProvider()) { - case CiProviders.cirrus: - return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}'; - case CiProviders.luci: - return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build? - } - return ''; -} - -String _getGitHash() { - switch(_getCiProvider()) { - case CiProviders.cirrus: - return Platform.environment['CIRRUS_CHANGE_IN_REPO']; - case CiProviders.luci: - return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI. - } - return ''; -} - -Future<void> _processTestOutput( - FlutterCompactFormatter formatter, - Stream<String> testOutput, - bq.TabledataResourceApi tableData, -) async { - final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) { - print('Processing...'); - }); - - await testOutput.forEach(formatter.processRawOutput); - heartbeat.cancel(); - formatter.finish(); - if (tableData == null || formatter.tests.isEmpty) { - return; - } - final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest(); - final String authors = await _getAuthors(); - request.rows = List<bq.TableDataInsertAllRequestRows>.from( - formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) => - bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> { - 'json': <String, dynamic>{ - 'source': <String, dynamic>{ - 'provider': _getCiProviderName(), - 'url': _getCiUrl(), - 'platform': <String, dynamic>{ - 'os': Platform.operatingSystem, - 'version': Platform.operatingSystemVersion, - }, - }, - 'test': <String, dynamic>{ - 'name': result.name, - 'result': result.status.toString(), - 'file': result.path, - 'line': result.line, - 'column': result.column, - 'time': result.totalTime, - }, - 'git': <String, dynamic>{ - 'author': authors, - 'pull_request': _getPrNumber(), - 'commit': _getGitHash(), - 'organization': 'flutter', - 'repository': 'flutter', - }, - 'error': result.status != TestStatus.failed ? null : <String, dynamic>{ - 'message': result.errorMessage, - 'stack_trace': result.stackTrace, - }, - 'information': result.messages, - }, - }), - ), - growable: false, - ); - final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci'); - if (response.insertErrors != null && response.insertErrors.isNotEmpty) { - print('${red}BigQuery insert errors:'); - print(response.toJson()); - print(reset); - } -} - -class EvalResult { - EvalResult({ - this.stdout, - this.stderr, - this.exitCode = 0, - }); - - final String stdout; - final String stderr; - final int exitCode; -} - -/// The number of Cirrus jobs that run web tests in parallel. -/// -/// WARNING: if you change this number, also change .cirrus.yml -/// and make sure it runs _all_ shards. -const int _kWebShardCount = 6; - -Future<void> _runFlutterWebTest(String workingDirectory, { - List<String> tests, -}) async { - List<String> allTests = <String>[]; - for (String testDirPath in tests) { - final Directory testDir = Directory(path.join(workingDirectory, testDirPath)); - allTests.addAll( - testDir.listSync(recursive: true) - .whereType<File>() - .where((File file) => file.path.endsWith('_test.dart')) - .map<String>((File file) => path.relative(file.path, from: workingDirectory)) - .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)), - ); - } - - // If a shard is specified only run tests in that shard. - final int webShard = int.tryParse(Platform.environment['WEB_SHARD'] ?? 'n/a'); - if (webShard != null) { - if (webShard >= _kWebShardCount) { - throw 'WEB_SHARD must be <= _kWebShardCount, but was $webShard'; - } - final List<String> shard = <String>[]; - for (int i = webShard; i < allTests.length; i += _kWebShardCount) { - shard.add(allTests[i]); - } - allTests = shard; - } - - print(allTests.join('\n')); - print('${allTests.length} tests total'); - - // Maximum number of tests to run in a single `flutter test`. We found that - // large batches can get flaky, possibly because we reuse a single instance - // of the browser, and after many tests the browser's state gets corrupted. - const int kBatchSize = 20; - List<String> batch = <String>[]; - for (int i = 0; i < allTests.length; i += 1) { - final String testFilePath = allTests[i]; - batch.add(testFilePath); - if (batch.length == kBatchSize || i == allTests.length - 1) { - await _runFlutterWebTestBatch(workingDirectory, batch: batch); - batch = <String>[]; - } - } -} - -Future<void> _runFlutterWebTestBatch(String workingDirectory, { - List<String> batch, -}) async { - final List<String> args = <String>[ - 'test', - if (_getCiProvider() == CiProviders.cirrus) - '--concurrency=1', // do not parallelize on Cirrus to reduce flakiness - '-v', - '--platform=chrome', - ...?flutterTestArgs, - ...batch, - ]; - - // TODO(jonahwilliams): fix relative path issues to make this unecessary. - final Directory oldCurrent = Directory.current; - Directory.current = Directory(path.join(flutterRoot, 'packages', 'flutter')); - try { - await runCommand( - flutter, - args, - workingDirectory: workingDirectory, - expectFlaky: false, - environment: <String, String>{ - 'FLUTTER_WEB': 'true', - 'FLUTTER_LOW_RESOURCE_MODE': 'true', - }, - ); - } finally { - Directory.current = oldCurrent; - } -} - Future<void> _runFlutterTest(String workingDirectory, { String script, bool expectFailure = false, @@ -790,8 +621,7 @@ Map<String, String> environment, List<String> tests = const <String>[], }) async { - assert(!printOutput || outputChecker == null, - 'Output either can be printed or checked but not both'); + assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); final List<String> args = <String>[ 'test', @@ -800,16 +630,15 @@ ]; final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage'); - if (shouldProcessOutput) { + if (shouldProcessOutput) args.add('--machine'); - } if (script != null) { final String fullScriptPath = path.join(workingDirectory, script); if (!FileSystemEntity.isFileSync(fullScriptPath)) { - print('Could not find test: $fullScriptPath'); - print('Working directory: $workingDirectory'); - print('Script: $script'); + print('${red}Could not find test$reset: $green$fullScriptPath$reset'); + print('Working directory: $cyan$workingDirectory$reset'); + print('Script: $green$script$reset'); if (!printOutput) print('This is one of the tests that does not normally print output.'); if (skip) @@ -876,110 +705,290 @@ } } -// the optional `file` argument is an override for testing -@visibleForTesting -Future<bool> verifyVersion(String filename, [File file]) async { - final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$'); - file ??= File(filename); - final String version = await file.readAsString(); - if (!file.existsSync()) { - print('$redLine'); - print('The version logic failed to create the Flutter version file.'); - print('$redLine'); - return false; - } - if (version == '0.0.0-unknown') { - print('$redLine'); - print('The version logic failed to determine the Flutter version.'); - print('$redLine'); - return false; - } - if (!version.contains(pattern)) { - print('$redLine'); - print('The version logic generated an invalid version string: "$version".'); - print('$redLine'); - return false; - } - return true; -} - -Future<void> _runIntegrationTests() async { - final String subShard = Platform.environment['SUBSHARD']; - - switch (subShard) { - case 'gradle1': - case 'gradle2': - // This runs some gradle integration tests if the subshard is Android. - await _androidGradleTests(subShard); - break; - default: - await _runDevicelabTest('dartdocs'); - - if (Platform.isLinux) { - await _runDevicelabTest('flutter_create_offline_test_linux'); - } else if (Platform.isWindows) { - await _runDevicelabTest('flutter_create_offline_test_windows'); - } else if (Platform.isMacOS) { - await _runDevicelabTest('flutter_create_offline_test_mac'); - await _runDevicelabTest('plugin_lint_mac'); -// TODO(jmagman): Re-enable once flakiness is resolved. -// await _runDevicelabTest('module_test_ios'); - } - } -} - -Future<void> _runDevicelabTest(String testName, {Map<String, String> env}) async { - await runCommand( - dart, - <String>['bin/run.dart', '-t', testName], - workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'), - environment: env, - ); -} - -String get androidSdkRoot { +Map<String, String> _initGradleEnvironment() { final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) ? Platform.environment['ANDROID_SDK_ROOT'] : Platform.environment['ANDROID_HOME']; if (androidSdkRoot == null || androidSdkRoot.isEmpty) { - return null; + print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset'); + exit(1); } - return androidSdkRoot; -} - -Future<void> _androidGradleTests(String subShard) async { - // TODO(dnfield): gradlew is crashing on the cirrus image and it's not clear why. - if (androidSdkRoot == null || Platform.isWindows) { - print('No Android SDK detected or on Windows, skipping Android gradle test.'); - return; - } - final Map<String, String> defaultEnv = <String, String>{ + return <String, String>{ 'ANDROID_HOME': androidSdkRoot, 'ANDROID_SDK_ROOT': androidSdkRoot, - 'ENABLE_ANDROID_EMBEDDING_V2': Platform.environment['ENABLE_ANDROID_EMBEDDING_V2'] ?? '', }; - if (subShard == 'gradle1') { - await _runDevicelabTest('gradle_plugin_light_apk_test', env: defaultEnv); - await _runDevicelabTest('gradle_plugin_fat_apk_test', env: defaultEnv); - await _runDevicelabTest('gradle_r8_test', env: defaultEnv); - await _runDevicelabTest('gradle_non_android_plugin_test', env: defaultEnv); - await _runDevicelabTest('gradle_jetifier_test', env: defaultEnv); +} + +final Map<String, String> gradleEnvironment = _initGradleEnvironment(); + +Future<void> _runHostOnlyDeviceLabTests() async { + if (Platform.isWindows) { + // TODO(ianh): remove when https://github.com/flutter/flutter/issues/36311 fixed by https://github.com/flutter/flutter/pull/42709 + return; } - if (subShard == 'gradle2') { - await _runDevicelabTest('gradle_plugin_bundle_test', env: defaultEnv); - await _runDevicelabTest('module_test', env: defaultEnv); - await _runDevicelabTest('module_host_with_custom_build_test', env: defaultEnv); - await _runDevicelabTest('build_aar_module_test', env: defaultEnv); - await _runDevicelabTest('plugin_test', env: defaultEnv); + + // Please don't add more tests here. We should not be using the devicelab + // logic to run tests outside devicelab, that's just confusing. + // Instead, create tests that are not devicelab tests, and run those. + + // TODO(ianh): Move the tests that are not running on devicelab any more out + // of the device lab directory. + + // List the tests to run. + // We split these into subshards. The tests are randomly distributed into + // those subshards so as to get a uniform distribution of costs, but the + // seed is fixed so that issues are reproducible. + final List<ShardRunner> tests = <ShardRunner>[ + // Keep this in alphabetical order. + () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: false), + if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'), + if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'), + if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'), + // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494 + if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: false), + if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true), + () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true), + // TODO(jmagman): Re-enable once flakiness is resolved, https://github.com/flutter/flutter/issues/37525 + // if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'), + if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'), + () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: false), + () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: true), + ]..shuffle(math.Random(0)); + + final int testsPerShard = tests.length ~/ kDeviceLabShardCount; + final Map<String, ShardRunner> subshards = <String, ShardRunner>{}; + + for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) { + String last = ''; + List<ShardRunner> sublist; + if (subshard < kDeviceLabShardCount - 1) { + sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard); + } else { + sublist = tests.sublist(subshard * testsPerShard, tests.length); + // We make sure the last shard ends in _last so it's easier to catch mismatches + // between `.cirrus.yml` and `test.dart`. + last = '_last'; + } + subshards['$subshard$last'] = () async { + for (ShardRunner test in sublist) + await test(); + }; + } + + await selectSubshard(subshards); +} + +Future<void> _runDevicelabTest(String testName, { + Map<String, String> environment, + bool testEmbeddingV2 = false, +}) async { + await runCommand( + dart, + <String>['bin/run.dart', '-t', testName], + workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'), + environment: <String, String>{ + ...?environment, + if (testEmbeddingV2) + 'ENABLE_ANDROID_EMBEDDING_V2': 'true', + }, + ); +} + +void deleteFile(String path) { + // This is technically a race condition but nobody else should be running + // while this script runs, so we should be ok. (Sadly recursive:true does not + // obviate the need for existsSync, at least on Windows.) + final File file = File(path); + if (file.existsSync()) + file.deleteSync(); +} + +enum CiProviders { + cirrus, + luci, +} + +Future<void> _processTestOutput( + FlutterCompactFormatter formatter, + Stream<String> testOutput, + bq.TabledataResourceApi tableData, +) async { + final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) { + print('Processing...'); + }); + + await testOutput.forEach(formatter.processRawOutput); + heartbeat.cancel(); + formatter.finish(); + if (tableData == null || formatter.tests.isEmpty) { + return; + } + final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest(); + final String authors = await _getAuthors(); + request.rows = List<bq.TableDataInsertAllRequestRows>.from( + formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) => + bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> { + 'json': <String, dynamic>{ + 'source': <String, dynamic>{ + 'provider': ciProviderName, + 'url': ciUrl, + 'platform': <String, dynamic>{ + 'os': Platform.operatingSystem, + 'version': Platform.operatingSystemVersion, + }, + }, + 'test': <String, dynamic>{ + 'name': result.name, + 'result': result.status.toString(), + 'file': result.path, + 'line': result.line, + 'column': result.column, + 'time': result.totalTime, + }, + 'git': <String, dynamic>{ + 'author': authors, + 'pull_request': prNumber, + 'commit': gitHash, + 'organization': 'flutter', + 'repository': 'flutter', + }, + 'error': result.status != TestStatus.failed ? null : <String, dynamic>{ + 'message': result.errorMessage, + 'stack_trace': result.stackTrace, + }, + 'information': result.messages, + }, + }), + ), + growable: false, + ); + final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci'); + if (response.insertErrors != null && response.insertErrors.isNotEmpty) { + print('${red}BigQuery insert errors:'); + print(response.toJson()); + print(reset); } } -Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard'); -Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard'); +CiProviders get ciProvider { + if (Platform.environment['CIRRUS_CI'] == 'true') { + return CiProviders.cirrus; + } + if (Platform.environment['LUCI_CONTEXT'] != null) { + return CiProviders.luci; + } + return null; +} -Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async { - final String item = Platform.environment[key]; - if (item != null) { +String get ciProviderName { + switch (ciProvider) { + case CiProviders.cirrus: + return 'cirrusci'; + case CiProviders.luci: + return 'luci'; + } + return 'unknown'; +} + +int get prNumber { + switch (ciProvider) { + case CiProviders.cirrus: + return Platform.environment['CIRRUS_PR'] == null + ? -1 + : int.tryParse(Platform.environment['CIRRUS_PR']); + case CiProviders.luci: + return -1; // LUCI doesn't know about this. + } + return -1; +} + +Future<String> _getAuthors() async { + final String exe = Platform.isWindows ? '.exe' : ''; + final String author = await runAndGetStdout( + 'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'], + workingDirectory: flutterRoot, + ).first; + return author; +} + +String get ciUrl { + switch (ciProvider) { + case CiProviders.cirrus: + return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}'; + case CiProviders.luci: + return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build? + } + return ''; +} + +String get gitHash { + switch(ciProvider) { + case CiProviders.cirrus: + return Platform.environment['CIRRUS_CHANGE_IN_REPO']; + case CiProviders.luci: + return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI. + } + return ''; +} + +/// Checks the given file's contents to determine if they match the allowed +/// pattern for version strings. +/// +/// Returns null if the contents are good. Returns a string if they are bad. +/// The string is an error message. +Future<String> verifyVersion(File file) async { + final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$'); + final String version = await file.readAsString(); + if (!file.existsSync()) + return 'The version logic failed to create the Flutter version file.'; + if (version == '0.0.0-unknown') + return 'The version logic failed to determine the Flutter version.'; + if (!version.contains(pattern)) + return 'The version logic generated an invalid version string: "$version".'; + return null; +} + +/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine +/// the shard and subshard (parsing it in the form shard-subshard-platform, ignoring +/// the platform). +/// +/// However, for local testing you can just set the SHARD and SUBSHARD +/// environment variables. For example, to run all the framework tests you can +/// just set SHARD=framework_tests. To run specifically the third subshard of +/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based). +Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0); +Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1); + +const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; + +Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async { + String item = Platform.environment[key]; + if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) { + final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-'); + assert(positionInTaskName < parts.length); + item = parts[positionInTaskName]; + } + if (item == null) { + for (String currentItem in items.keys) { + print('$bold$key=$currentItem$reset'); + await items[currentItem](); + print(''); + } + } else { if (!items.containsKey(item)) { print('${red}Invalid $name: $item$reset'); print('The available ${name}s are: ${items.keys.join(", ")}'); @@ -987,11 +996,5 @@ } print('$bold$key=$item$reset'); await items[item](); - } else { - for (String currentItem in items.keys) { - print('$bold$key=$currentItem$reset'); - await items[currentItem](); - print(''); - } } }
diff --git a/dev/bots/test/bot_test.dart b/dev/bots/test/bot_test.dart new file mode 100644 index 0000000..476280c --- /dev/null +++ b/dev/bots/test/bot_test.dart
@@ -0,0 +1,13 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'common.dart'; + +void main() { + test('BOT variable is set on bots', () { + expect(Platform.environment['BOT'], 'true'); + }); +}
diff --git a/dev/bots/test/sdk_directory_has_space_test.dart b/dev/bots/test/sdk_directory_has_space_test.dart new file mode 100644 index 0000000..f48dbdb --- /dev/null +++ b/dev/bots/test/sdk_directory_has_space_test.dart
@@ -0,0 +1,21 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import 'common.dart'; + +void main() { + test('We are in a directory with a space in it', () async { + // The Flutter SDK should be in a directory with a space in it, to make sure + // our tools support that. + final String expectedName = Platform.environment['FLUTTER_SDK_PATH_WITH_SPACE']; + expect(expectedName, 'flutter sdk'); + expect(expectedName, contains(' ')); + final List<String> parts = path.split(Directory.current.absolute.path); + expect(parts.reversed.take(3), <String>['bots', 'dev', expectedName]); + }); +}
diff --git a/dev/bots/test/test_test.dart b/dev/bots/test/test_test.dart index a3abf6a..31df83f 100644 --- a/dev/bots/test/test_test.dart +++ b/dev/bots/test/test_test.dart
@@ -29,7 +29,11 @@ ]; for (String version in valid_versions) { when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version)); - expect(await verifyVersion(version, file), isTrue, reason: '$version is invalid'); + expect( + await verifyVersion(file), + isNull, + reason: '$version is valid but verifyVersionFile said it was bad', + ); } }); @@ -41,10 +45,15 @@ '1.2.3-pre', '1.2.3-pre.1+hotfix.1', ' 1.2.3', + '1.2.3-hotfix.1', ]; for (String version in invalid_versions) { when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version)); - expect(await verifyVersion(version, file), isFalse); + expect( + await verifyVersion(file), + 'The version logic generated an invalid version string: "$version".', + reason: '$version is invalid but verifyVersionFile said it was fine', + ); } }); });
diff --git a/packages/flutter_tools/dart_test.yaml b/packages/flutter_tools/dart_test.yaml index cd8c697..789a255 100644 --- a/packages/flutter_tools/dart_test.yaml +++ b/packages/flutter_tools/dart_test.yaml
@@ -1,5 +1,9 @@ -tags: - "no_coverage": - "create": - "integration": - +# package:test configuration +# https://github.com/dart-lang/test/blob/master/pkgs/test/doc/configuration.md + +# Some of our tests take an absurdly long time to run, and on some +# hosts they can take even longer due to the host suddenly being +# overloaded. For this reason, we set the test timeout to +# significantly more than it would be by default, and we never set the +# timeouts in the tests themselves. +timeout: 15m
diff --git a/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart index abad543..4cf3b17 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart
@@ -135,7 +135,7 @@ return _updateIdeConfig( expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('creates non-existent files', () async { final Map<String, String> templateManifest = _getManifest( @@ -155,7 +155,7 @@ return _updateIdeConfig( expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('overwrites existing files with --overwrite', () async { final Map<String, String> templateManifest = _getManifest( @@ -181,7 +181,7 @@ args: <String>['--overwrite'], expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('only adds new templates without --overwrite', () async { final Map<String, String> templateManifest = _getManifest( @@ -212,7 +212,7 @@ args: <String>['--update-templates'], expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('update all templates with --overwrite', () async { final Map<String, String> templateManifest = _getManifest( @@ -239,7 +239,7 @@ args: <String>['--update-templates', '--overwrite'], expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('removes deleted imls with --overwrite', () async { final Map<String, String> templateManifest = _getManifest( @@ -275,7 +275,7 @@ args: <String>['--update-templates', '--overwrite'], expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); testUsingContext('removes deleted imls with --overwrite, including empty parent dirs', () async { final Map<String, String> templateManifest = _getManifest( @@ -316,7 +316,7 @@ args: <String>['--update-templates', '--overwrite'], expectedContents: expectedContents, ); - }, timeout: const Timeout.factor(2.0)); + }); }); }
diff --git a/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart b/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart index 38b8980..b1c2a8a 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart
@@ -16,9 +16,6 @@ import '../../src/common.dart'; import '../../src/context.dart'; -/// Test case timeout for tests involving project analysis. -const Timeout allowForSlowAnalyzeTests = Timeout.factor(5.0); - final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false; final Map<Type, Generator> noColorTerminalOverride = <Type, Generator>{ Platform: _kNoColorTerminalPlatform, @@ -54,7 +51,7 @@ ], ); expect(libMain.existsSync(), isTrue); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -65,7 +62,7 @@ arguments: <String>['analyze'], statusTextContains: <String>['No issues found!'], ); - }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -80,6 +77,7 @@ }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); + // Analyze in the current directory - no arguments testUsingContext('working directory with errors', () async { // Break the code to produce the "The parameter 'onPressed' is required" hint @@ -110,7 +108,7 @@ exitMessageContains: '2 issues found.', toolExit: true, ); - }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), ...noColorTerminalOverride, }); @@ -140,7 +138,7 @@ exitMessageContains: '3 issues found.', toolExit: true, ); - }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), ...noColorTerminalOverride });
diff --git a/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart index 224d406..8f0476e 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart
@@ -73,7 +73,7 @@ expect(await command.usageValues, containsPair(CustomDimensions.commandBuildBundleIsModule, 'true')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('bundle getUsage indicate that project is not a module', () async { final String projectPath = await createProject(tempDir, @@ -83,7 +83,7 @@ expect(await command.usageValues, containsPair(CustomDimensions.commandBuildBundleIsModule, 'false')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('bundle getUsage indicate the target platform', () async { final String projectPath = await createProject(tempDir, @@ -93,7 +93,7 @@ expect(await command.usageValues, containsPair(CustomDimensions.commandBuildBundleTargetPlatform, 'android-arm')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('bundle fails to build for Windows if feature is disabled', () async { fs.file('lib/main.dart').createSync(recursive: true);
diff --git a/packages/flutter_tools/test/commands.shard/permeable/create_test.dart b/packages/flutter_tools/test/commands.shard/permeable/create_test.dart index 2056e49..6165895 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/create_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/create_test.dart
@@ -78,7 +78,7 @@ ], ); return _runFlutterTest(projectDir); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -96,7 +96,7 @@ 'ios/Runner/GeneratedPluginRegistrant.h', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -118,11 +118,9 @@ 'ios/', ]); return _runFlutterTest(projectDir); - }, - timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ - Pub: () => const Pub(), - }, - ); + }, overrides: <Type, Generator>{ + Pub: () => const Pub(), + }); testUsingContext('cannot create a project if non-empty non-project directory exists with .metadata', () async { await projectDir.absolute.childDirectory('blag').create(recursive: true); @@ -138,7 +136,7 @@ '.ios/', ]), throwsToolExit(message: 'Sorry, unable to detect the type of project to recreate')); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), ...noColorTerminalOverride, }); @@ -164,7 +162,7 @@ '.ios/', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -189,7 +187,7 @@ '.ios/', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -214,7 +212,7 @@ 'lib/flutter_project.dart', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -245,7 +243,7 @@ 'test/widget_test.dart', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -267,7 +265,7 @@ 'ios/Runner/main.m', ], ); - }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -297,7 +295,7 @@ ], ); return _runFlutterTest(projectDir); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -319,7 +317,7 @@ ], ); return _runFlutterTest(projectDir.childDirectory('example')); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -346,7 +344,7 @@ 'example/ios/Runner/main.m', ], ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('plugin project with custom org', () async { return _createProject( @@ -366,7 +364,7 @@ 'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java', ], ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('plugin project with valid custom project name', () async { return _createProject( @@ -386,7 +384,7 @@ 'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java', ], ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('plugin project with invalid custom project name', () async { expect( @@ -396,7 +394,7 @@ ), throwsToolExit(message: '"xyz.xyz" is not a valid Dart package name.'), ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('legacy app project with-driver-test', () async { return _createAndAnalyzeProject( @@ -404,7 +402,7 @@ <String>['--with-driver-test', '--template=app'], <String>['lib/main.dart'], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -439,7 +437,7 @@ 'android/', 'ios/', ]); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -463,7 +461,7 @@ final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString(); expect(actualContents.contains('useAndroidX'), true); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('non androidx app project', () async { Cache.flutterRoot = '../..'; @@ -484,7 +482,7 @@ final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString(); expect(actualContents.contains('useAndroidX'), false); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('androidx is used by default in a module project', () async { Cache.flutterRoot = '../..'; @@ -501,7 +499,7 @@ project.usesAndroidX, true, ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('non androidx module', () async { Cache.flutterRoot = '../..'; @@ -518,7 +516,7 @@ project.usesAndroidX, false, ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('androidx is used by default in a plugin project', () async { Cache.flutterRoot = '../..'; @@ -539,7 +537,7 @@ final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString(); expect(actualContents.contains('useAndroidX'), true); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('non androidx plugin project', () async { Cache.flutterRoot = '../..'; @@ -560,7 +558,7 @@ final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString(); expect(actualContents.contains('useAndroidX'), false); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('app supports macOS if requested', () async { Cache.flutterRoot = '../..'; @@ -573,7 +571,7 @@ await runner.run(<String>['create', '--no-pub', '--macos', projectDir.path]); expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), true); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('app does not include macOS by default', () async { Cache.flutterRoot = '../..'; @@ -586,7 +584,7 @@ await runner.run(<String>['create', '--no-pub', projectDir.path]); expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), false); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('plugin supports macOS if requested', () async { Cache.flutterRoot = '../..'; @@ -599,7 +597,7 @@ await runner.run(<String>['create', '--no-pub', '--template=plugin', '--macos', projectDir.path]); expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), true); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('plugin does not include macOS by default', () async { Cache.flutterRoot = '../..'; @@ -612,7 +610,7 @@ await runner.run(<String>['create', '--no-pub', '--template=plugin', projectDir.path]); expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), false); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('has correct content and formatting with module template', () async { Cache.flutterRoot = '../..'; @@ -708,7 +706,7 @@ }, overrides: <Type, Generator>{ FlutterVersion: () => mockFlutterVersion, Platform: _kNoColorTerminalPlatform, - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('has correct content and formatting with app template', () async { Cache.flutterRoot = '../..'; @@ -779,7 +777,7 @@ }, overrides: <Type, Generator>{ FlutterVersion: () => mockFlutterVersion, Platform: _kNoColorTerminalPlatform, - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('has correct application id for android and bundle id for ios', () async { Cache.flutterRoot = '../..'; @@ -827,7 +825,7 @@ }, overrides: <Type, Generator>{ FlutterVersion: () => mockFlutterVersion, Platform: _kNoColorTerminalPlatform, - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen default template over existing project', () async { Cache.flutterRoot = '../..'; @@ -841,7 +839,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: app\n')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen default template over existing app project with no metadta and detect the type', () async { Cache.flutterRoot = '../..'; @@ -858,7 +856,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: app\n')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen app template over existing app project and detect the type', () async { Cache.flutterRoot = '../..'; @@ -872,7 +870,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: app\n')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen template over existing module project and detect the type', () async { Cache.flutterRoot = '../..'; @@ -886,7 +884,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: module\n')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen default template over existing plugin project and detect the type', () async { Cache.flutterRoot = '../..'; @@ -900,7 +898,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: plugin')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen default template over existing package project and detect the type', () async { Cache.flutterRoot = '../..'; @@ -914,7 +912,7 @@ final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync(); expect(metadata, contains('project_type: package')); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen module .android/ folder, reusing custom org', () async { await _createProject( @@ -930,7 +928,7 @@ '.android/app/src/main/java/com/bar/foo/flutter_project/host/MainActivity.java', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -947,7 +945,7 @@ await project.ios.productBundleIdentifier, 'com.bar.foo.flutterProject', ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -974,7 +972,7 @@ 'android/app/src/main/java/com/example/flutter_project/MainActivity.java', ], ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen app ios/ folder, reusing custom org', () async { await _createProject( @@ -989,7 +987,7 @@ await project.ios.productBundleIdentifier, 'com.bar.foo.flutterProject', ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('can re-gen plugin ios/ and example/ folders, reusing custom org', () async { await _createProject( @@ -1022,7 +1020,7 @@ await project.example.ios.productBundleIdentifier, 'com.bar.foo.flutterProjectExample', ); - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('fails to re-gen without specified org when org is ambiguous', () async { await _createProject( @@ -1040,7 +1038,7 @@ () => _createProject(projectDir, <String>[], <String>[]), throwsToolExit(message: 'Ambiguous organization'), ); - }, timeout: allowForCreateFlutterProject); + }); // Verify that we help the user correct an option ordering issue testUsingContext('produces sensible error message', () async { @@ -1102,7 +1100,7 @@ 'ios/Runner/GeneratedPluginRegistrant.h', ], ); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -1128,7 +1126,6 @@ expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub'))); expect(loggingProcessManager.commands.first, contains('--offline')); }, - timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ ProcessManager: () => loggingProcessManager, Pub: () => const Pub(), @@ -1147,7 +1144,6 @@ expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub'))); expect(loggingProcessManager.commands.first, isNot(contains('--offline'))); }, - timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ ProcessManager: () => loggingProcessManager, Pub: () => const Pub(), @@ -1168,7 +1164,7 @@ ); expect(projectDir.childDirectory('lib').childFile('main.dart').readAsStringSync(), contains('void main() {}')); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ HttpClientFactory: () => () => MockHttpClient(200, result: 'void main() {}'), });
diff --git a/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart index 07da823..a9197ed 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart
@@ -207,7 +207,7 @@ expectDependenciesResolved(projectPath); expectZeroPluginsInjected(projectPath); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -220,7 +220,7 @@ expectDependenciesResolved(projectPath); expectZeroPluginsInjected(projectPath); - }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -234,7 +234,7 @@ expect(await getCommand.usageValues, containsPair(CustomDimensions.commandPackagesNumberPlugins, '0')); - }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -248,7 +248,7 @@ expect(await getCommand.usageValues, containsPair(CustomDimensions.commandPackagesProjectModule, 'false')); - }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -262,7 +262,7 @@ expect(await getCommand.usageValues, containsPair(CustomDimensions.commandPackagesProjectModule, 'true')); - }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -275,7 +275,7 @@ expectDependenciesResolved(projectPath); expectZeroPluginsInjected(projectPath); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -288,7 +288,7 @@ expectDependenciesResolved(projectPath); expectModulePluginInjected(projectPath); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); @@ -309,7 +309,7 @@ expectDependenciesResolved(exampleProjectPath); expectPluginInjected(exampleProjectPath); - }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{ + }, overrides: <Type, Generator>{ Pub: () => const Pub(), }); });
diff --git a/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart b/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart index 1591e75..a93b50e 100644 --- a/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart +++ b/packages/flutter_tools/test/general.shard/commands/build_aar_test.dart
@@ -57,7 +57,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('indicate that project is a plugin', () async { final String projectPath = await createProject(tempDir, @@ -69,7 +69,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('indicate the target platform', () async { final String projectPath = await createProject(tempDir, @@ -82,7 +82,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); }); group('Gradle', () {
diff --git a/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart b/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart index a67c9fa..05fa1ff 100644 --- a/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart +++ b/packages/flutter_tools/test/general.shard/commands/build_apk_test.dart
@@ -48,7 +48,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('split per abi', () async { final String projectPath = await createProject(tempDir, @@ -65,7 +65,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('build type', () async { final String projectPath = await createProject(tempDir, @@ -92,7 +92,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); }); group('Gradle', () { @@ -223,8 +223,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), GradleUtils: () => GradleUtils(), ProcessManager: () => mockProcessManager, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('shrinking is disabled when --no-shrink is passed', () async { final String projectPath = await createProject(tempDir, @@ -255,8 +254,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), GradleUtils: () => GradleUtils(), ProcessManager: () => mockProcessManager, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('guides the user when the shrinker fails', () async { final String projectPath = await createProject(tempDir, @@ -312,8 +310,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('reports when the app isn\'t using AndroidX', () async { final String projectPath = await createProject(tempDir, @@ -366,8 +363,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('reports when the app is using AndroidX', () async { final String projectPath = await createProject(tempDir, @@ -422,8 +418,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); }); }
diff --git a/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart b/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart index a00d279..7663156 100644 --- a/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart +++ b/packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart
@@ -48,7 +48,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); testUsingContext('build type', () async { final String projectPath = await createProject(tempDir, @@ -75,7 +75,7 @@ }, overrides: <Type, Generator>{ AndroidBuilder: () => FakeAndroidBuilder(), - }, timeout: allowForCreateFlutterProject); + }); }); group('Gradle', () { @@ -212,8 +212,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), GradleUtils: () => GradleUtils(), ProcessManager: () => mockProcessManager, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('shrinking is disabled when --no-shrink is passed', () async { final String projectPath = await createProject( @@ -246,8 +245,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), GradleUtils: () => GradleUtils(), ProcessManager: () => mockProcessManager, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('guides the user when the shrinker fails', () async { final String projectPath = await createProject(tempDir, @@ -303,8 +301,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('reports when the app isn\'t using AndroidX', () async { final String projectPath = await createProject(tempDir, @@ -357,8 +354,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); testUsingContext('reports when the app is using AndroidX', () async { final String projectPath = await createProject(tempDir, @@ -413,8 +409,7 @@ FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir), ProcessManager: () => mockProcessManager, Usage: () => mockUsage, - }, - timeout: allowForCreateFlutterProject); + }); }); }
diff --git a/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart b/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart index f1ce5ba..aee21d0 100644 --- a/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart +++ b/packages/flutter_tools/test/integration.shard/daemon_mode_test.dart
@@ -2,13 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) import 'dart:async'; import 'dart:convert'; +import 'dart:io'; import 'package:file/file.dart'; import 'package:flutter_tools/src/base/file_system.dart'; @@ -21,79 +17,58 @@ import 'test_utils.dart'; void main() { - group('daemon_mode', () { - Directory tempDir; + test('device.getDevices', () async { + final Directory tempDir = createResolvedTempDirectorySync('daemon_mode_test.'); + final BasicProject _project = BasicProject(); - Process process; + await _project.setUpIn(tempDir); - setUp(() async { - tempDir = createResolvedTempDirectorySync('daemon_mode_test.'); - await _project.setUpIn(tempDir); - }); + final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter'); - tearDown(() async { - tryToDelete(tempDir); - process?.kill(); - }); + const ProcessManager processManager = LocalProcessManager(); + final Process process = await processManager.start( + <String>[flutterBin, '--show-test-device', 'daemon'], + workingDirectory: tempDir.path, + ); - test('device.getDevices', () async { - final String flutterBin = - fs.path.join(getFlutterRoot(), 'bin', 'flutter'); + final StreamController<String> stdout = StreamController<String>.broadcast(); + transformToLines(process.stdout).listen((String line) => stdout.add(line)); + final Stream<Map<String, dynamic>> stream = stdout + .stream + .map<Map<String, dynamic>>(parseFlutterResponse) + .where((Map<String, dynamic> value) => value != null); - const ProcessManager processManager = LocalProcessManager(); - process = await processManager.start( - <String>[flutterBin, '--show-test-device', 'daemon'], - workingDirectory: tempDir.path); + Map<String, dynamic> response = await stream.first; + expect(response['event'], 'daemon.connected'); - final StreamController<String> stdout = - StreamController<String>.broadcast(); + // start listening for devices + process.stdin.writeln('[${jsonEncode(<String, dynamic>{ + 'id': 1, + 'method': 'device.enable', + })}]'); + response = await stream.first; + expect(response['id'], 1); + expect(response['error'], isNull); - transformToLines(process.stdout) - .listen((String line) => stdout.add(line)); + // [{"event":"device.added","params":{"id":"flutter-tester","name": + // "Flutter test device","platform":"flutter-tester","emulator":false}}] + response = await stream.first; + expect(response['event'], 'device.added'); - final Stream<Map<String, dynamic>> stream = - stdout.stream.where((String line) { - final Map<String, dynamic> response = parseFlutterResponse(line); - // ignore 'Starting device daemon...' - if (response == null) { - return false; - } - // TODO(devoncarew): Remove this after #25440 lands. - if (response['event'] == 'daemon.showMessage') { - return false; - } - return true; - }).map(parseFlutterResponse); + // get the list of all devices + process.stdin.writeln('[${jsonEncode(<String, dynamic>{ + 'id': 2, + 'method': 'device.getDevices', + })}]'); + response = await stream.first; + expect(response['id'], 2); + expect(response['error'], isNull); - Map<String, dynamic> response = await stream.first; - expect(response['event'], 'daemon.connected'); + final dynamic result = response['result']; + expect(result, isList); + expect(result, isNotEmpty); - // start listening for devices - process.stdin.writeln('[${jsonEncode(<String, dynamic>{ - 'id': 1, - 'method': 'device.enable', - })}]'); - response = await stream.first; - expect(response['id'], 1); - expect(response['error'], isNull); - - // [{"event":"device.added","params":{"id":"flutter-tester","name": - // "Flutter test device","platform":"flutter-tester","emulator":false}}] - response = await stream.first; - expect(response['event'], 'device.added'); - - // get the list of all devices - process.stdin.writeln('[${jsonEncode(<String, dynamic>{ - 'id': 2, - 'method': 'device.getDevices', - })}]'); - response = await stream.first; - expect(response['id'], 2); - expect(response['error'], isNull); - - final dynamic result = response['result']; - expect(result, isList); - expect(result, isNotEmpty); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // This test uses the `flutter` tool, which could be blocked behind the startup lock for a long time. + tryToDelete(tempDir); + process.kill(); + }); }
diff --git a/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart b/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart index a53cff9..e6a743c 100644 --- a/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart +++ b/packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart
@@ -2,11 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) +import 'dart:io'; + import 'package:file/file.dart'; import 'package:flutter_tools/src/base/file_system.dart'; @@ -16,43 +13,37 @@ import 'test_utils.dart'; void main() { - group('debugger', () { - Directory tempDir; + test('can step over statements', () async { + final Directory tempDir = createResolvedTempDirectorySync('debugger_stepping_test.'); + final SteppingProject _project = SteppingProject(); - FlutterRunTestDriver _flutter; + await _project.setUpIn(tempDir); - setUp(() async { - tempDir = createResolvedTempDirectorySync('debugger_stepping_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterRunTestDriver(tempDir); - }); + final FlutterRunTestDriver _flutter = FlutterRunTestDriver(tempDir); - tearDown(() async { - await _flutter.stop(); - tryToDelete(tempDir); - }); + await _flutter.run(withDebugger: true, startPaused: true); + await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine); + await _flutter.resume(); + await _flutter.waitForPause(); // Now we should be on the breakpoint. - test('can step over statements', () async { - await _flutter.run(withDebugger: true, startPaused: true); - await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine); - await _flutter.resume(); - await _flutter.waitForPause(); // Now we should be on the breakpoint. + expect((await _flutter.getSourceLocation()).line, equals(_project.breakpointLine)); - expect((await _flutter.getSourceLocation()).line, equals(_project.breakpointLine)); + // Issue 5 steps, ensuring that we end up on the annotated lines each time. + for (int i = 1; i <= _project.numberOfSteps; i += 1) { + await _flutter.stepOverOrOverAsyncSuspension(); + final SourcePosition location = await _flutter.getSourceLocation(); + final int actualLine = location.line; - // Issue 5 steps, ensuring that we end up on the annotated lines each time. - for (int i = 1; i <= _project.numberOfSteps; i += 1) { - await _flutter.stepOverOrOverAsyncSuspension(); - final SourcePosition location = await _flutter.getSourceLocation(); - final int actualLine = location.line; + // Get the line we're expected to stop at by searching for the comment + // within the source code. + final int expectedLine = _project.lineForStep(i); - // Get the line we're expected to stop at by searching for the comment - // within the source code. - final int expectedLine = _project.lineForStep(i); + expect(actualLine, equals(expectedLine), + reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine' + ); + } - expect(actualLine, equals(expectedLine), - reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine'); - } - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + await _flutter.stop(); + tryToDelete(tempDir); + }); }
diff --git a/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart b/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart index a15dad4..00038a8 100644 --- a/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart +++ b/packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart
@@ -2,12 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) import 'dart:async'; +import 'dart:io'; import 'package:file/file.dart'; import 'package:flutter_tools/src/base/file_system.dart'; @@ -20,120 +16,133 @@ import 'test_driver.dart'; import 'test_utils.dart'; -void main() { - group('flutter run expression evaluation', () { - Directory tempDir; - final BasicProject _project = BasicProject(); - FlutterRunTestDriver _flutter; +void batch1() { + final BasicProject _project = BasicProject(); + Directory tempDir; + FlutterRunTestDriver _flutter; - setUp(() async { - tempDir = createResolvedTempDirectorySync('run_expression_eval_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterRunTestDriver(tempDir); - }); + Future<void> initProject() async { + tempDir = createResolvedTempDirectorySync('run_expression_eval_test.'); + await _project.setUpIn(tempDir); + _flutter = FlutterRunTestDriver(tempDir); + } - tearDown(() async { - await _flutter.stop(); - tryToDelete(tempDir); - }); + Future<void> cleanProject() async { + await _flutter.stop(); + tryToDelete(tempDir); + } - Future<void> breakInBuildMethod(FlutterTestDriver flutter) async { - await _flutter.breakAt( - _project.buildMethodBreakpointUri, - _project.buildMethodBreakpointLine, - ); - } + Future<void> breakInBuildMethod(FlutterTestDriver flutter) async { + await _flutter.breakAt( + _project.buildMethodBreakpointUri, + _project.buildMethodBreakpointLine, + ); + } - Future<void> breakInTopLevelFunction(FlutterTestDriver flutter) async { - await _flutter.breakAt( - _project.topLevelFunctionBreakpointUri, - _project.topLevelFunctionBreakpointLine, - ); - } + Future<void> breakInTopLevelFunction(FlutterTestDriver flutter) async { + await _flutter.breakAt( + _project.topLevelFunctionBreakpointUri, + _project.topLevelFunctionBreakpointLine, + ); + } - test('can evaluate trivial expressions in top level function', () async { - await _flutter.run(withDebugger: true); - await breakInTopLevelFunction(_flutter); - await evaluateTrivialExpressions(_flutter); - }); + test('flutter run expression evaluation - can evaluate trivial expressions in top level function', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInTopLevelFunction(_flutter); + await evaluateTrivialExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate trivial expressions in build method', () async { - await _flutter.run(withDebugger: true); - await breakInBuildMethod(_flutter); - await evaluateTrivialExpressions(_flutter); - }); + test('flutter run expression evaluation - can evaluate trivial expressions in build method', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInBuildMethod(_flutter); + await evaluateTrivialExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate complex expressions in top level function', () async { - await _flutter.run(withDebugger: true); - await breakInTopLevelFunction(_flutter); - await evaluateComplexExpressions(_flutter); - }); + test('flutter run expression evaluation - can evaluate complex expressions in top level function', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInTopLevelFunction(_flutter); + await evaluateComplexExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate complex expressions in build method', () async { - await _flutter.run(withDebugger: true); - await breakInBuildMethod(_flutter); - await evaluateComplexExpressions(_flutter); - }); + test('flutter run expression evaluation - can evaluate complex expressions in build method', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInBuildMethod(_flutter); + await evaluateComplexExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate expressions returning complex objects in top level function', () async { - await _flutter.run(withDebugger: true); - await breakInTopLevelFunction(_flutter); - await evaluateComplexReturningExpressions(_flutter); - }); + test('flutter run expression evaluation - can evaluate expressions returning complex objects in top level function', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInTopLevelFunction(_flutter); + await evaluateComplexReturningExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate expressions returning complex objects in build method', () async { - await _flutter.run(withDebugger: true); - await breakInBuildMethod(_flutter); - await evaluateComplexReturningExpressions(_flutter); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('flutter run expression evaluation - can evaluate expressions returning complex objects in build method', () async { + await initProject(); + await _flutter.run(withDebugger: true); + await breakInBuildMethod(_flutter); + await evaluateComplexReturningExpressions(_flutter); + await cleanProject(); + }); +} - group('flutter test expression evaluation', () { - Directory tempDir; - final TestsProject _project = TestsProject(); - FlutterTestTestDriver _flutter; +void batch2() { + final TestsProject _project = TestsProject(); + Directory tempDir; + FlutterTestTestDriver _flutter; - setUp(() async { - tempDir = createResolvedTempDirectorySync('test_expression_eval_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterTestTestDriver(tempDir); - }); + Future<void> initProject() async { + tempDir = createResolvedTempDirectorySync('test_expression_eval_test.'); + await _project.setUpIn(tempDir); + _flutter = FlutterTestTestDriver(tempDir); + } - tearDown(() async { - await _flutter.quit(); - tryToDelete(tempDir); - }); + Future<void> cleanProject() async { + await _flutter?.quit(); + tryToDelete(tempDir); + } - test('can evaluate trivial expressions in a test', () async { - await _flutter.test( - withDebugger: true, - beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), - ); - await _flutter.waitForPause(); - await evaluateTrivialExpressions(_flutter); - await _flutter.resume(); - }); + test('flutter test expression evaluation - can evaluate trivial expressions in a test', () async { + await initProject(); + await _flutter.test( + withDebugger: true, + beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), + ); + await _flutter.waitForPause(); + await evaluateTrivialExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate complex expressions in a test', () async { - await _flutter.test( - withDebugger: true, - beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), - ); - await _flutter.waitForPause(); - await evaluateComplexExpressions(_flutter); - await _flutter.resume(); - }); + test('flutter test expression evaluation - can evaluate complex expressions in a test', () async { + await initProject(); + await _flutter.test( + withDebugger: true, + beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), + ); + await _flutter.waitForPause(); + await evaluateComplexExpressions(_flutter); + await cleanProject(); + }); - test('can evaluate expressions returning complex objects in a test', () async { - await _flutter.test( - withDebugger: true, - beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), - ); - await _flutter.waitForPause(); - await evaluateComplexReturningExpressions(_flutter); - await _flutter.resume(); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('flutter test expression evaluation - can evaluate expressions returning complex objects in a test', () async { + await initProject(); + await _flutter.test( + withDebugger: true, + beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine), + ); + await _flutter.waitForPause(); + await evaluateComplexReturningExpressions(_flutter); + await cleanProject(); + }); } Future<void> evaluateTrivialExpressions(FlutterTestDriver flutter) async { @@ -164,3 +173,8 @@ final InstanceRef res = await flutter.evaluate(resp.id, r'"$year-$month-$day"'); expect(res.valueAsString, equals('${now.year}-${now.month}-${now.day}')); } + +void main() { + batch1(); + batch2(); +}
diff --git a/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart b/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart index b5592c1..2d868b3 100644 --- a/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart +++ b/packages/flutter_tools/test/integration.shard/flutter_attach_test.dart
@@ -2,11 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) +import 'dart:io'; + import 'package:file/file.dart'; import 'package:flutter_tools/src/base/file_system.dart'; @@ -33,35 +30,36 @@ tryToDelete(tempDir); }); - group('attached process', () { - test('writes pid-file', () async { - final File pidFile = tempDir.childFile('test.pid'); - await _flutterRun.run(withDebugger: true); - await _flutterAttach.attach( - _flutterRun.vmServicePort, - pidFile: pidFile, - ); - expect(pidFile.existsSync(), isTrue); - }); - test('can hot reload', () async { - await _flutterRun.run(withDebugger: true); - await _flutterAttach.attach(_flutterRun.vmServicePort); - await _flutterAttach.hotReload(); - }); - test('can detach, reattach, hot reload', () async { - await _flutterRun.run(withDebugger: true); - await _flutterAttach.attach(_flutterRun.vmServicePort); - await _flutterAttach.detach(); - await _flutterAttach.attach(_flutterRun.vmServicePort); - await _flutterAttach.hotReload(); - }); - test('killing process behaves the same as detach ', () async { - await _flutterRun.run(withDebugger: true); - await _flutterAttach.attach(_flutterRun.vmServicePort); - await _flutterAttach.quit(); - _flutterAttach = FlutterRunTestDriver(tempDir, logPrefix: 'ATTACH-2'); - await _flutterAttach.attach(_flutterRun.vmServicePort); - await _flutterAttach.hotReload(); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('writes pid-file', () async { + final File pidFile = tempDir.childFile('test.pid'); + await _flutterRun.run(withDebugger: true); + await _flutterAttach.attach( + _flutterRun.vmServicePort, + pidFile: pidFile, + ); + expect(pidFile.existsSync(), isTrue); + }); + + test('can hot reload', () async { + await _flutterRun.run(withDebugger: true); + await _flutterAttach.attach(_flutterRun.vmServicePort); + await _flutterAttach.hotReload(); + }); + + test('can detach, reattach, hot reload', () async { + await _flutterRun.run(withDebugger: true); + await _flutterAttach.attach(_flutterRun.vmServicePort); + await _flutterAttach.detach(); + await _flutterAttach.attach(_flutterRun.vmServicePort); + await _flutterAttach.hotReload(); + }); + + test('killing process behaves the same as detach ', () async { + await _flutterRun.run(withDebugger: true); + await _flutterAttach.attach(_flutterRun.vmServicePort); + await _flutterAttach.quit(); + _flutterAttach = FlutterRunTestDriver(tempDir, logPrefix: 'ATTACH-2'); + await _flutterAttach.attach(_flutterRun.vmServicePort); + await _flutterAttach.hotReload(); + }); }
diff --git a/packages/flutter_tools/test/integration.shard/flutter_run_test.dart b/packages/flutter_tools/test/integration.shard/flutter_run_test.dart index cc83323..32f50e1 100644 --- a/packages/flutter_tools/test/integration.shard/flutter_run_test.dart +++ b/packages/flutter_tools/test/integration.shard/flutter_run_test.dart
@@ -2,11 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) import 'package:file/file.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/io.dart'; @@ -18,47 +13,45 @@ import 'test_utils.dart'; void main() { - group('flutter_run', () { - Directory tempDir; - final BasicProject _project = BasicProject(); - FlutterRunTestDriver _flutter; + Directory tempDir; + final BasicProject _project = BasicProject(); + FlutterRunTestDriver _flutter; - setUp(() async { - tempDir = createResolvedTempDirectorySync('run_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterRunTestDriver(tempDir); - }); + setUp(() async { + tempDir = createResolvedTempDirectorySync('run_test.'); + await _project.setUpIn(tempDir); + _flutter = FlutterRunTestDriver(tempDir); + }); - tearDown(() async { - await _flutter.stop(); - tryToDelete(tempDir); - }); + tearDown(() async { + await _flutter.stop(); + tryToDelete(tempDir); + }); - test('reports an error if an invalid device is supplied', () async { - // This test forces flutter to check for all possible devices to catch issues - // like https://github.com/flutter/flutter/issues/21418 which were skipped - // over because other integration tests run using flutter-tester which short-cuts - // some of the checks for devices. - final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter'); + test('flutter run reports an error if an invalid device is supplied', () async { + // This test forces flutter to check for all possible devices to catch issues + // like https://github.com/flutter/flutter/issues/21418 which were skipped + // over because other integration tests run using flutter-tester which short-cuts + // some of the checks for devices. + final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter'); - const ProcessManager _processManager = LocalProcessManager(); - final ProcessResult _proc = await _processManager.run( - <String>[flutterBin, 'run', '-d', 'invalid-device-id'], - workingDirectory: tempDir.path, - ); + const ProcessManager _processManager = LocalProcessManager(); + final ProcessResult _proc = await _processManager.run( + <String>[flutterBin, 'run', '-d', 'invalid-device-id'], + workingDirectory: tempDir.path, + ); - expect(_proc.stdout, isNot(contains('flutter has exited unexpectedly'))); - expect(_proc.stderr, isNot(contains('flutter has exited unexpectedly'))); - if (!_proc.stderr.toString().contains('Unable to locate a development') - && !_proc.stdout.toString().contains('No devices found with name or id matching')) { - fail("'flutter run -d invalid-device-id' did not produce the expected error"); - } - }); + expect(_proc.stdout, isNot(contains('flutter has exited unexpectedly'))); + expect(_proc.stderr, isNot(contains('flutter has exited unexpectedly'))); + if (!_proc.stderr.toString().contains('Unable to locate a development') + && !_proc.stdout.toString().contains('No devices found with name or id matching')) { + fail("'flutter run -d invalid-device-id' did not produce the expected error"); + } + }); - test('writes pid-file', () async { - final File pidFile = tempDir.childFile('test.pid'); - await _flutter.run(pidFile: pidFile); - expect(pidFile.existsSync(), isTrue); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('flutter run writes pid-file', () async { + final File pidFile = tempDir.childFile('test.pid'); + await _flutter.run(pidFile: pidFile); + expect(pidFile.existsSync(), isTrue); + }); }
diff --git a/packages/flutter_tools/test/integration.shard/hot_reload_test.dart b/packages/flutter_tools/test/integration.shard/hot_reload_test.dart index a651294..2d7d3cd 100644 --- a/packages/flutter_tools/test/integration.shard/hot_reload_test.dart +++ b/packages/flutter_tools/test/integration.shard/hot_reload_test.dart
@@ -2,11 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) import 'dart:async'; import 'package:file/file.dart'; @@ -20,139 +15,137 @@ import 'test_utils.dart'; void main() { - group('hot reload tests', () { - Directory tempDir; - final HotReloadProject _project = HotReloadProject(); - FlutterRunTestDriver _flutter; + Directory tempDir; + final HotReloadProject _project = HotReloadProject(); + FlutterRunTestDriver _flutter; - setUp(() async { - tempDir = createResolvedTempDirectorySync('hot_reload_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterRunTestDriver(tempDir); - }); + setUp(() async { + tempDir = createResolvedTempDirectorySync('hot_reload_test.'); + await _project.setUpIn(tempDir); + _flutter = FlutterRunTestDriver(tempDir); + }); - tearDown(() async { - await _flutter?.stop(); - tryToDelete(tempDir); - }); + tearDown(() async { + await _flutter?.stop(); + tryToDelete(tempDir); + }); - test('hot reload works without error', () async { - await _flutter.run(); + test('hot reload works without error', () async { + await _flutter.run(); + await _flutter.hotReload(); + }); + + test('newly added code executes during hot reload', () async { + await _flutter.run(); + _project.uncommentHotReloadPrint(); + final StringBuffer stdout = StringBuffer(); + final StreamSubscription<String> subscription = _flutter.stdout.listen(stdout.writeln); + try { await _flutter.hotReload(); - }); - - test('newly added code executes during hot reload', () async { - await _flutter.run(); - _project.uncommentHotReloadPrint(); - final StringBuffer stdout = StringBuffer(); - final StreamSubscription<String> subscription = _flutter.stdout.listen(stdout.writeln); - try { - await _flutter.hotReload(); - expect(stdout.toString(), contains('(((((RELOAD WORKED)))))')); - } finally { - await subscription.cancel(); - } - }); - - test('hot restart works without error', () async { - await _flutter.run(); - await _flutter.hotRestart(); - }); - - test('breakpoints are hit after hot reload', () async { - Isolate isolate; - await _flutter.run(withDebugger: true, startPaused: true); - final Completer<void> sawTick1 = Completer<void>(); - final Completer<void> sawTick3 = Completer<void>(); - final Completer<void> sawDebuggerPausedMessage = Completer<void>(); - final StreamSubscription<String> subscription = _flutter.stdout.listen( - (String line) { - if (line.contains('((((TICK 1))))')) { - expect(sawTick1.isCompleted, isFalse); - sawTick1.complete(); - } - if (line.contains('((((TICK 3))))')) { - expect(sawTick3.isCompleted, isFalse); - sawTick3.complete(); - } - if (line.contains('The application is paused in the debugger on a breakpoint.')) { - expect(sawDebuggerPausedMessage.isCompleted, isFalse); - sawDebuggerPausedMessage.complete(); - } - }, - ); - await _flutter.resume(); // we start paused so we can set up our TICK 1 listener before the app starts - unawaited(sawTick1.future.timeout( - const Duration(seconds: 5), - onTimeout: () { print('The test app is taking longer than expected to print its synchronization line...'); }, - )); - await sawTick1.future; // after this, app is in steady state - await _flutter.addBreakpoint( - _project.scheduledBreakpointUri, - _project.scheduledBreakpointLine, - ); - await _flutter.hotReload(); // reload triggers code which eventually hits the breakpoint - isolate = await _flutter.waitForPause(); - expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); - await _flutter.resume(); - await _flutter.addBreakpoint( - _project.buildBreakpointUri, - _project.buildBreakpointLine, - ); - bool reloaded = false; - final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; }); - await sawTick3.future; // this should happen before it pauses - isolate = await _flutter.waitForPause(); - expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); - await sawDebuggerPausedMessage.future; - expect(reloaded, isFalse); - await _flutter.resume(); - await reloadFuture; - expect(reloaded, isTrue); - reloaded = false; + expect(stdout.toString(), contains('(((((RELOAD WORKED)))))')); + } finally { await subscription.cancel(); - }); + } + }); - test('hot reload doesn\'t reassemble if paused', () async { - await _flutter.run(withDebugger: true); - final Completer<void> sawTick2 = Completer<void>(); - final Completer<void> sawTick3 = Completer<void>(); - final Completer<void> sawDebuggerPausedMessage1 = Completer<void>(); - final Completer<void> sawDebuggerPausedMessage2 = Completer<void>(); - final StreamSubscription<String> subscription = _flutter.stdout.listen( - (String line) { - if (line.contains('((((TICK 2))))')) { - expect(sawTick2.isCompleted, isFalse); - sawTick2.complete(); - } - if (line.contains('The application is paused in the debugger on a breakpoint.')) { - expect(sawDebuggerPausedMessage1.isCompleted, isFalse); - sawDebuggerPausedMessage1.complete(); - } - if (line.contains('The application is paused in the debugger on a breakpoint; interface might not update.')) { - expect(sawDebuggerPausedMessage2.isCompleted, isFalse); - sawDebuggerPausedMessage2.complete(); - } - }, - ); - await _flutter.addBreakpoint( - _project.buildBreakpointUri, - _project.buildBreakpointLine, - ); - bool reloaded = false; - final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; }); - await sawTick2.future; // this should happen before it pauses - final Isolate isolate = await _flutter.waitForPause(); - expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); - expect(reloaded, isFalse); - await sawDebuggerPausedMessage1.future; // this is the one where it say "uh, you broke into the debugger while reloading" - await reloadFuture; // this is the one where it times out because you're in the debugger - expect(reloaded, isTrue); - await _flutter.hotReload(); // now we're already paused - expect(sawTick3.isCompleted, isFalse); - await sawDebuggerPausedMessage2.future; // so we just get told that nothing is going to happen - await _flutter.resume(); - await subscription.cancel(); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('hot restart works without error', () async { + await _flutter.run(); + await _flutter.hotRestart(); + }); + + test('breakpoints are hit after hot reload', () async { + Isolate isolate; + await _flutter.run(withDebugger: true, startPaused: true); + final Completer<void> sawTick1 = Completer<void>(); + final Completer<void> sawTick3 = Completer<void>(); + final Completer<void> sawDebuggerPausedMessage = Completer<void>(); + final StreamSubscription<String> subscription = _flutter.stdout.listen( + (String line) { + if (line.contains('((((TICK 1))))')) { + expect(sawTick1.isCompleted, isFalse); + sawTick1.complete(); + } + if (line.contains('((((TICK 3))))')) { + expect(sawTick3.isCompleted, isFalse); + sawTick3.complete(); + } + if (line.contains('The application is paused in the debugger on a breakpoint.')) { + expect(sawDebuggerPausedMessage.isCompleted, isFalse); + sawDebuggerPausedMessage.complete(); + } + }, + ); + await _flutter.resume(); // we start paused so we can set up our TICK 1 listener before the app starts + unawaited(sawTick1.future.timeout( + const Duration(seconds: 5), + onTimeout: () { print('The test app is taking longer than expected to print its synchronization line...'); }, + )); + await sawTick1.future; // after this, app is in steady state + await _flutter.addBreakpoint( + _project.scheduledBreakpointUri, + _project.scheduledBreakpointLine, + ); + await _flutter.hotReload(); // reload triggers code which eventually hits the breakpoint + isolate = await _flutter.waitForPause(); + expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); + await _flutter.resume(); + await _flutter.addBreakpoint( + _project.buildBreakpointUri, + _project.buildBreakpointLine, + ); + bool reloaded = false; + final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; }); + await sawTick3.future; // this should happen before it pauses + isolate = await _flutter.waitForPause(); + expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); + await sawDebuggerPausedMessage.future; + expect(reloaded, isFalse); + await _flutter.resume(); + await reloadFuture; + expect(reloaded, isTrue); + reloaded = false; + await subscription.cancel(); + }); + + test('hot reload doesn\'t reassemble if paused', () async { + await _flutter.run(withDebugger: true); + final Completer<void> sawTick2 = Completer<void>(); + final Completer<void> sawTick3 = Completer<void>(); + final Completer<void> sawDebuggerPausedMessage1 = Completer<void>(); + final Completer<void> sawDebuggerPausedMessage2 = Completer<void>(); + final StreamSubscription<String> subscription = _flutter.stdout.listen( + (String line) { + if (line.contains('((((TICK 2))))')) { + expect(sawTick2.isCompleted, isFalse); + sawTick2.complete(); + } + if (line.contains('The application is paused in the debugger on a breakpoint.')) { + expect(sawDebuggerPausedMessage1.isCompleted, isFalse); + sawDebuggerPausedMessage1.complete(); + } + if (line.contains('The application is paused in the debugger on a breakpoint; interface might not update.')) { + expect(sawDebuggerPausedMessage2.isCompleted, isFalse); + sawDebuggerPausedMessage2.complete(); + } + }, + ); + await _flutter.addBreakpoint( + _project.buildBreakpointUri, + _project.buildBreakpointLine, + ); + bool reloaded = false; + final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; }); + await sawTick2.future; // this should happen before it pauses + final Isolate isolate = await _flutter.waitForPause(); + expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint)); + expect(reloaded, isFalse); + await sawDebuggerPausedMessage1.future; // this is the one where it say "uh, you broke into the debugger while reloading" + await reloadFuture; // this is the one where it times out because you're in the debugger + expect(reloaded, isTrue); + await _flutter.hotReload(); // now we're already paused + expect(sawTick3.isCompleted, isFalse); + await sawDebuggerPausedMessage2.future; // so we just get told that nothing is going to happen + await _flutter.resume(); + await subscription.cancel(); + }); }
diff --git a/packages/flutter_tools/test/integration.shard/lifetime_test.dart b/packages/flutter_tools/test/integration.shard/lifetime_test.dart index 9e5284a..1ff3152 100644 --- a/packages/flutter_tools/test/integration.shard/lifetime_test.dart +++ b/packages/flutter_tools/test/integration.shard/lifetime_test.dart
@@ -2,11 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// Integration tests which invoke flutter instead of unit testing the code -// will not produce meaningful coverage information - we can measure coverage -// from the isolate running the test, but not from the isolate started via -// the command line process. -@Tags(<String>['no_coverage']) import 'dart:async'; import 'package:file/file.dart'; @@ -23,32 +18,30 @@ const Duration requiredLifespan = Duration(seconds: 5); void main() { - group('flutter run', () { - final BasicProject _project = BasicProject(); - FlutterRunTestDriver _flutter; - Directory tempDir; + final BasicProject _project = BasicProject(); + FlutterRunTestDriver _flutter; + Directory tempDir; - setUp(() async { - tempDir = createResolvedTempDirectorySync('lifetime_test.'); - await _project.setUpIn(tempDir); - _flutter = FlutterRunTestDriver(tempDir); - }); + setUp(() async { + tempDir = createResolvedTempDirectorySync('lifetime_test.'); + await _project.setUpIn(tempDir); + _flutter = FlutterRunTestDriver(tempDir); + }); - tearDown(() async { - await _flutter.stop(); - tryToDelete(tempDir); - }); + tearDown(() async { + await _flutter.stop(); + tryToDelete(tempDir); + }); - test('does not terminate when a debugger is attached', () async { - await _flutter.run(withDebugger: true); - await Future<void>.delayed(requiredLifespan); - expect(_flutter.hasExited, equals(false)); - }); + test('flutter run does not terminate when a debugger is attached', () async { + await _flutter.run(withDebugger: true); + await Future<void>.delayed(requiredLifespan); + expect(_flutter.hasExited, equals(false)); + }); - test('does not terminate when a debugger is attached and pause-on-exceptions', () async { - await _flutter.run(withDebugger: true, pauseOnExceptions: true); - await Future<void>.delayed(requiredLifespan); - expect(_flutter.hasExited, equals(false)); - }); - }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow. + test('fluter run does not terminate when a debugger is attached and pause-on-exceptions', () async { + await _flutter.run(withDebugger: true, pauseOnExceptions: true); + await Future<void>.delayed(requiredLifespan); + expect(_flutter.hasExited, equals(false)); + }); }
diff --git a/packages/flutter_tools/test/src/common.dart b/packages/flutter_tools/test/src/common.dart index 7741daf..00de957 100644 --- a/packages/flutter_tools/test/src/common.dart +++ b/packages/flutter_tools/test/src/common.dart
@@ -127,13 +127,6 @@ return projectPath; } -/// Test case timeout for tests involving remote calls to `pub get` or similar. -const Timeout allowForRemotePubInvocation = Timeout.factor(10.0); - -/// Test case timeout for tests involving creating a Flutter project with -/// `--no-pub`. Use [allowForRemotePubInvocation] when creation involves `pub`. -const Timeout allowForCreateFlutterProject = Timeout.factor(3.0); - Future<void> expectToolExitLater(Future<dynamic> future, Matcher messageMatcher) async { try { await future;
diff --git a/packages/flutter_tools/test/src/context.dart b/packages/flutter_tools/test/src/context.dart index 57e4465..7f1fdff 100644 --- a/packages/flutter_tools/test/src/context.dart +++ b/packages/flutter_tools/test/src/context.dart
@@ -48,7 +48,6 @@ void testUsingContext( String description, dynamic testMethod(), { - Timeout timeout, Map<Type, Generator> overrides = const <Type, Generator>{}, bool initializeFlutterRoot = true, String testOn, @@ -137,8 +136,7 @@ }, ); }); - }, timeout: timeout ?? const Timeout(Duration(seconds: 60)), - testOn: testOn, skip: skip); + }, testOn: testOn, skip: skip); } void _printBufferedErrors(AppContext testContext) {