Adding win devicelab bot results to dashboard. (#80974) (#81077)
* Adding win devicelab bot results to dashboard. (#80974)
* Update prod_builders.json
* Update prod_builders.json and remove agent manifest.yaml
Changes to switch off agent results, which are no longer being used.
Co-authored-by: yusufm <mohsinally@google.com>
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml
deleted file mode 100644
index 86806f7..0000000
--- a/dev/devicelab/manifest.yaml
+++ /dev/null
@@ -1,349 +0,0 @@
-# Describes the tasks we run in the continuous integration (CI) environment.
-#
-# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
-# every master commit.
-#
-# [1] github.com/flutter/cocoon
-
-# CI tasks.
-#
-# Each key in this dictionary is the unique name of a task, which also
-# corresponds to a file in the "bin/" directory that the task runner will run.
-#
-# Required properties:
-#
-# * description: free form string
-# describes what the task is about
-# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
-# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
-# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
-# run device tests on red Cirrus builds.
-# * required_agent_capabilities: a list of strings
-# list of capabilities a devicelab agent must have to be able to run this task.
-#
-# Optional properties:
-#
-# * flaky: boolean true or false
-# whether the task is considered flaky; the result of running a flaky task does not affect
-# the overall build status. Tests should be marked flaky when newly added, until they have
-# been proved for a few cycles. Tests that are actually flaky but not being actively worked
-# on should be hidden because they just cause confusion otherwise.
-# * on_luci: boolean true or false
-# whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the
-# devicelab to LUCI migration. And when the migration is complete, on_luci flag will be
-# removed.
-
-tasks:
- # Tests of compiling in a variety of modes
-
- complex_layout_ios__compile:
- description: >
- Collects various performance metrics of compiling the Complex
- Layout sample app for iOS from Mac.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- complex_layout_win__compile:
- description: >
- Collects various performance metrics of compiling the Complex
- Layout for Android from Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- basic_material_app_ios__compile:
- description: >
- Collects various performance metrics of compiling the default
- app for iOS from Mac.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- basic_material_app_win__compile:
- description: >
- Collects various performance metrics of compiling the default
- app for Android from Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- flutter_gallery_ios__compile:
- description: >
- Collects various performance metrics of compiling the Flutter
- Gallery for iOS from Mac.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- flutter_gallery_win__compile:
- description: >
- Collects various performance metrics of compiling the Flutter
- Gallery for Android from Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- windows_chrome_dev_mode:
- description: >
- Run flutter web on the devicelab and hot restart.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- # iOS on-device tests
-
- flutter_gallery_sksl_warmup__transition_perf_e2e_ios32:
- description: >
- Measures the runtime performance of Flutter gallery transitions on iPhone4s
- with SkSL shader warm-up with e2e.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios32"]
- flaky: true
-
- ios_defines_test:
- description: >
- Builds a Framework with a --dart-define and verifies it can be used as a constant
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- ios_content_validation_test:
- description: >
- Builds an obfuscated app and verifies contents and structure
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
- on_luci: true
-
- tiles_scroll_perf_ios__timeline_summary:
- description: >
- Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- platform_views_scroll_perf_ios__timeline_summary:
- description: >
- Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- flutter_gallery_ios32__start_up:
- description: >
- Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios32"]
-
- flavors_test_ios:
- description: >
- Checks that flavored builds work on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- external_ui_integration_test_ios:
- description: >
- Checks that external UIs work on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- channels_integration_test_ios:
- description: >
- Checks that platform channels work on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- platform_interaction_test_ios:
- description: >
- Checks platform interaction on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- platform_channel_sample_test_ios:
- description: >
- Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- platform_channel_sample_test_swift:
- description: >
- Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- platform_view_ios__start_up:
- description: >
- Verifies that Platform View can be used from an iOS project.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- backdrop_filter_perf_ios__timeline_summary:
- description: >
- Measures the runtime performance of backdrop filter blurs on iOS.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- post_backdrop_filter_perf_ios__timeline_summary:
- description: >
- Measures the runtime performance of animations after a backdrop filter is removed on iOS.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- complex_layout_scroll_perf_ios__timeline_summary:
- description: >
- Measures the runtime performance of the Complex Layout sample app on
- iOS.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- flutter_gallery_ios__start_up:
- description: >
- Measures the startup time of the Flutter Gallery app on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- complex_layout_ios__start_up:
- description: >
- Measures the startup time of the Complex Layout sample app on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- flutter_gallery_ios__transition_perf:
- description: >
- Measures the performance of screen transitions in Flutter Gallery on
- iOS.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- hello_world_ios__compile:
- description: >
- Measures the IPA size of Hello World.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- microbenchmarks_ios:
- description: >
- Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
- stage: devicelab_ios
- #TODO(godofredoc): Remove ios/12 capability once
- # https://github.com/flutter/flutter/issues/49635 is fixed.
- required_agent_capabilities: ["mac/ios", "ios/12"]
-
- flutter_view_ios__start_up:
- description: >
- Verifies that Flutter View can be used from an iOS project.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- integration_ui_ios_driver:
- description: >
- Runs end-to-end Flutter tests on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- integration_ui_ios_keyboard_resize:
- description: >
- Runs end-to-end Flutter tests on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- integration_ui_ios_screenshot:
- description: >
- Runs end-to-end Flutter tests on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- integration_ui_ios_textfield:
- description: >
- Runs end-to-end Flutter tests on iPhone 6.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- ios_platform_view_tests:
- description: >
- Runs end-to-end tests with platform views in the scene.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- macos_chrome_dev_mode:
- description: >
- Run flutter web on the devicelab and hot restart.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- simple_animation_perf_ios:
- description: >
- Measure CPU/GPU usage percentages of a simple animation.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
- description: >
- A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
- stage: devicelab_ios
- required_agent_capabilities: ["mac-catalina/ios"]
-
- smoke_catalina_start_up:
- description: >
- A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
- stage: devicelab
- required_agent_capabilities: ["mac-catalina/android"]
-
- # macOS target platform tests
- hot_mode_dev_cycle_macos_target__benchmark:
- description: >
- Checks the functionality and performance of hot reload on a macOS target platform
- stage: devicelab
- required_agent_capabilities: ["mac/ios"]
-
- ios_app_with_extensions_test:
- description: >
- Checks that an iOS app with extensions can be built for physical and simulated devices.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- # Tests running on Windows host
-
- flavors_test_win:
- description: >
- Checks that flavored builds work on Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- channels_integration_test_win:
- description: >
- Checks that platform channels work when app is launched from Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- hot_mode_dev_cycle_win__benchmark:
- description: >
- Measures the performance of Dart VM hot patching feature on Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows/android"]
-
- # Tests running on Linux hosts
-
- flutter_gallery__transition_perf_e2e_ios32:
- description: >
- Measures the performance of screen transitions in Flutter Gallery on
- Android with e2e.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios32"]
- flaky: true
-
- flutter_gallery__transition_perf_e2e_ios:
- description: >
- Measures the performance of screen transitions in Flutter Gallery on
- Android with e2e.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- new_gallery_ios__transition_perf:
- description: >
- Measures the performance of screen transitions in the new Flutter Gallery on iOS.
- stage: devicelab_ios
- required_agent_capabilities: ["mac/ios"]
-
- large_image_changer_perf_ios:
- description: >
- Measures memory, cpu, and gpu usage when rotating through a series of large images.
- stage: devicelab
- required_agent_capabilities: ["mac/ios"]
-
- # android_splash_screen_integration_test:
- # description: >
- # Runs end-to-end test of Flutter's Android splash behavior.
- # stage: devicelab
- # required_agent_capabilities: ["linux/android"]
- # flaky: true
diff --git a/dev/devicelab/test/manifest_test.dart b/dev/devicelab/test/manifest_test.dart
deleted file mode 100644
index c0cc172..0000000
--- a/dev/devicelab/test/manifest_test.dart
+++ /dev/null
@@ -1,207 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:flutter_devicelab/framework/manifest.dart';
-
-import 'common.dart';
-
-void main() {
- group('production manifest', () {
- test('must be valid', () {
- final Manifest manifest = loadTaskManifest();
- expect(manifest.tasks, isNotEmpty);
-
- final ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'simple_animation_perf_ios');
- expect(task.description, 'Measure CPU/GPU usage percentages of a simple animation.\n');
- expect(task.stage, 'devicelab_ios');
- expect(task.requiredAgentCapabilities, <String>['mac/ios']);
-
- for (final ManifestTask task in manifest.tasks) {
- final File taskFile = File('bin/tasks/${task.name}.dart');
- expect(taskFile.existsSync(), true,
- reason: 'File ${taskFile.path} corresponding to manifest task "${task.name}" not found');
- }
- });
- });
-
- group('manifest parser', () {
- void testManifestError(
- String testDescription,
- String errorMessage,
- String yaml,
- ) {
- test(testDescription, () {
- try {
- loadTaskManifest(yaml);
- } on ManifestError catch (error) {
- expect(error.message, errorMessage);
- }
- });
- }
-
- test('accepts task with minimum amount of configuration', () {
- final Manifest manifest = loadTaskManifest('''
-tasks:
- minimum_configuration_task:
- description: Description is mandatory.
- stage: stage_is_mandatory_too
- required_agent_capabilities: ["so-is-capability"]
-''');
-
- expect(manifest.tasks.single.description, 'Description is mandatory.');
- expect(manifest.tasks.single.stage, 'stage_is_mandatory_too');
- expect(manifest.tasks.single.requiredAgentCapabilities, <String>['so-is-capability']);
- expect(manifest.tasks.single.isFlaky, false);
- expect(manifest.tasks.single.timeoutInMinutes, null);
- });
-
- testManifestError(
- 'invalid top-level type',
- 'Manifest must be a dictionary but was YamlScalar: null',
- '',
- );
-
- testManifestError(
- 'invalid top-level key',
- 'Unrecognized property "bad" in manifest. Allowed properties: tasks',
- '''
- bad:
- key: yes
- ''',
- );
-
- testManifestError(
- 'invalid tasks list type',
- 'Value of "tasks" must be a dictionary but was YamlList: [a, b]',
- '''
- tasks:
- - a
- - b
- ''',
- );
-
- testManifestError(
- 'invalid task name type',
- 'Task name must be a string but was int: 1',
- '''
- tasks:
- 1: 2
- ''',
- );
-
- testManifestError(
- 'invalid task type',
- 'Value of task "foo" must be a dictionary but was int: 2',
- '''
- tasks:
- foo: 2
- ''',
- );
-
- testManifestError(
- 'invalid task property',
- 'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities, flaky, timeout_in_minutes, on_luci',
- '''
- tasks:
- foo:
- bar: 2
- ''',
- );
-
- testManifestError(
- 'invalid required_agent_capabilities type',
- 'required_agent_capabilities must be a list but was int: 1',
- '''
- tasks:
- foo:
- required_agent_capabilities: 1
- ''',
- );
-
- testManifestError(
- 'invalid required_agent_capabilities element type',
- 'required_agent_capabilities[0] must be a string but was int: 1',
- '''
- tasks:
- foo:
- required_agent_capabilities: [1]
- ''',
- );
-
- testManifestError(
- 'missing description',
- 'Task description must not be empty in task "foo".',
- '''
- tasks:
- foo:
- required_agent_capabilities: ["a"]
- ''',
- );
-
- testManifestError(
- 'missing stage',
- 'Task stage must not be empty in task "foo".',
- '''
- tasks:
- foo:
- description: b
- required_agent_capabilities: ["a"]
- ''',
- );
-
- testManifestError(
- 'missing required_agent_capabilities',
- 'requiredAgentCapabilities must not be empty in task "foo".',
- '''
- tasks:
- foo:
- description: b
- stage: c
- required_agent_capabilities: []
- ''',
- );
-
- testManifestError(
- 'bad flaky type',
- 'flaky must be a boolean but was String: not-a-boolean',
- '''
- tasks:
- foo:
- description: b
- stage: c
- required_agent_capabilities: ["a"]
- flaky: not-a-boolean
- ''',
- );
-
- test('accepts boolean flaky option', () {
- final Manifest manifest = loadTaskManifest('''
-tasks:
- flaky_task:
- description: d
- stage: s
- required_agent_capabilities: ["c"]
- flaky: true
-''');
-
- expect(manifest.tasks.single.name, 'flaky_task');
- expect(manifest.tasks.single.isFlaky, isTrue);
- });
-
- test('accepts custom timeout_in_minutes option', () {
- final Manifest manifest = loadTaskManifest('''
-tasks:
- task_with_custom_timeout:
- description: d
- stage: s
- required_agent_capabilities: ["c"]
- timeout_in_minutes: 120
-''');
-
- expect(manifest.tasks.single.timeoutInMinutes, 120);
- });
- });
-}
diff --git a/dev/prod_builders.json b/dev/prod_builders.json
index 1f005ee..e4609ce 100644
--- a/dev/prod_builders.json
+++ b/dev/prod_builders.json
@@ -863,6 +863,48 @@
"repo": "flutter",
"task_name": "win_web_tool_tests",
"flaky": true
+ },
+ {
+ "name": "Windows_android complex_layout_win__compile",
+ "repo": "flutter",
+ "task_name": "complex_layout_win__compile",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android basic_material_app_win__compile",
+ "repo": "flutter",
+ "task_name": "basic_material_app_win__compile",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android flutter_gallery_win__compile",
+ "repo": "flutter",
+ "task_name": "flutter_gallery_win__compile",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android windows_chrome_dev_mode",
+ "repo": "flutter",
+ "task_name": "windows_chrome_dev_mode",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android flavors_test_win",
+ "repo": "flutter",
+ "task_name": "flavors_test_win",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android channels_integration_test_win",
+ "repo": "flutter",
+ "task_name": "channels_integration_test_win",
+ "flaky": false
+ },
+ {
+ "name": "Windows_android hot_mode_dev_cycle_win__benchmark",
+ "repo": "flutter",
+ "task_name": "hot_mode_dev_cycle_win__benchmark",
+ "flaky": false
}
]
}
