Add a builder recipe for flutter engine builds. This is one of the first steps to split engine recipes in smaller tasks with a goal of running all of them in less than 30 mins. Change-Id: I2cfffc7267ad5ca6cba96603dd994531ed848037 Bug: https://github.com/flutter/flutter/issues/81855 Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/13740 Reviewed-by: Zach Anderson <zra@google.com> Reviewed-by: Yegor Jbanov <yjbanov@google.com> Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipes/engine_v2/builder.expected/basic.json b/recipes/engine_v2/builder.expected/basic.json new file mode 100644 index 0000000..47ed335 --- /dev/null +++ b/recipes/engine_v2/builder.expected/basic.json
@@ -0,0 +1,449 @@ +[ + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CACHE]/builder/src/out" + ], + "infra_step": true, + "name": "Clobber build output" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Ensure checkout cache" + }, + { + "cmd": [], + "name": "Checkout source code" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Checkout source code.Clobber cache", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Checkout source code.Ensure checkout cache", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src/flutter', 'url': 'https://chromium.googlesource.com/external/github.com/flutter/engine'}]", + "--revision_mapping_file", + "{\"got_engine_revision\": \"src/flutter\"}", + "--git-cache-dir", + "[CACHE]/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src/flutter@HEAD" + ], + "cwd": "[CACHE]/builder", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "102400", + "GIT_HTTP_LOW_SPEED_TIME": "1800" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0", + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]", + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "Checkout source code.bot_update", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision_cp\": \"refs/heads/master@{#84512}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_engine_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@SET_BUILD_PROPERTY@got_engine_revision_cp@\"refs/heads/master@{#84512}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[CACHE]/builder", + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]", + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "Checkout source code.gclient runhooks", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "ensure goma" + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/goma/client", + "-ensure-file", + "fuchsia/third_party/goma/client/${platform} release", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "infra_step": true, + "name": "ensure goma.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/third_party/goma/client/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "[CACHE]/builder/src/flutter/tools/gn", + "--goma", + "--ios" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client" + }, + "name": "gn --ios" + }, + { + "cmd": [], + "name": "setup goma" + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "restart" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "setup goma.start goma", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "RECIPE_REPO[depot_tools]/ninja", + "-j", + "100", + "-C", + "[CACHE]/builder/src/out/ios_debug" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "build ios_debug" + }, + { + "cmd": [], + "name": "teardown goma" + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "jsonstatus", + "[CACHE]/goma/client/jsonstatus" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.goma jsonstatus", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"notice\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"infra_status\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"num_user_error\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ping_status_code\": 200@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "stat" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.goma stats", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "stop" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.stop goma", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CLEANUP]/compiler_proxy.WARNING", + "/path/to/tmp/" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "teardown goma.read goma_client warning log", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@compiler_proxy.WARNING@test log@@@", + "@@@STEP_LOG_END@compiler_proxy.WARNING@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CACHE]/goma/client/goma_stats.json", + "/path/to/tmp/" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "teardown goma.read goma_stats.json", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "[CACHE]/builder/src/script1.sh" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "name": "generator1" + }, + { + "cmd": [ + "[CACHE]/builder/src/myscript.sh", + "param1", + "param2" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "name": "mytest" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipes/engine_v2/builder.expected/mac.json b/recipes/engine_v2/builder.expected/mac.json new file mode 100644 index 0000000..d9c6c79 --- /dev/null +++ b/recipes/engine_v2/builder.expected/mac.json
@@ -0,0 +1,513 @@ +[ + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CACHE]/builder/src/out" + ], + "infra_step": true, + "name": "Clobber build output" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Ensure checkout cache" + }, + { + "cmd": [], + "name": "Checkout source code" + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Checkout source code.Clobber cache", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/builder" + ], + "infra_step": true, + "name": "Checkout source code.Ensure checkout cache", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src/flutter', 'url': 'https://chromium.googlesource.com/external/github.com/flutter/engine'}]", + "--revision_mapping_file", + "{\"got_engine_revision\": \"src/flutter\"}", + "--git-cache-dir", + "[CACHE]/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src/flutter@HEAD" + ], + "cwd": "[CACHE]/builder", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "102400", + "GIT_HTTP_LOW_SPEED_TIME": "1800" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0", + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]", + "RECIPE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "Checkout source code.bot_update", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"HEAD\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision_cp\": \"refs/heads/master@{#84512}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_engine_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@SET_BUILD_PROPERTY@got_engine_revision_cp@\"refs/heads/master@{#84512}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[CACHE]/builder", + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]", + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "Checkout source code.gclient runhooks", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/osx_sdk", + "-ensure-file", + "infra/tools/mac_toolchain/${platform} git_revision:9a1adc55bf4a1173784da3ba2f8cb06421606748", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "infra_step": true, + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:9a1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/mac_toolchain/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[CACHE]/osx_sdk/mac_toolchain", + "install", + "-kind", + "ios", + "-xcode-version", + "9f2000", + "-output-dir", + "[CACHE]/osx_sdk/XCode.app", + "-cipd-package-prefix", + "flutter_internal/ios/xcode" + ], + "infra_step": true, + "name": "install xcode" + }, + { + "cmd": [ + "sudo", + "xcode-select", + "--switch", + "[CACHE]/osx_sdk/XCode.app" + ], + "infra_step": true, + "name": "select XCode" + }, + { + "cmd": [], + "name": "ensure goma" + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/goma/client", + "-ensure-file", + "fuchsia/third_party/goma/client/${platform} release", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "infra_step": true, + "name": "ensure goma.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/third_party/goma/client/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "[CACHE]/builder/src/flutter/tools/gn", + "--goma", + "--ios" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client" + }, + "name": "gn --ios" + }, + { + "cmd": [], + "name": "setup goma" + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "restart" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "setup goma.start goma", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "RECIPE_REPO[depot_tools]/ninja", + "-j", + "100", + "-C", + "[CACHE]/builder/src/out/ios_debug" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "build ios_debug" + }, + { + "cmd": [], + "name": "teardown goma" + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "jsonstatus", + "[CACHE]/goma/client/jsonstatus" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.goma jsonstatus", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"notice\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"infra_status\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"num_user_error\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"ping_status_code\": 200@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "stat" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.goma stats", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CACHE]/goma/client/goma_ctl.py", + "stop" + ], + "env": { + "GLOG_log_dir": "[CLEANUP]", + "GOMA_CACHE_DIR": "[CACHE]/goma", + "GOMA_DEPS_CACHE_FILE": "goma_deps_cache", + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json", + "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache", + "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256", + "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog", + "GOMA_STORE_LOCAL_RUN_OUTPUT": "True", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "name": "teardown goma.stop goma", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CLEANUP]/compiler_proxy.WARNING", + "/path/to/tmp/" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "teardown goma.read goma_client warning log", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@compiler_proxy.WARNING@test log@@@", + "@@@STEP_LOG_END@compiler_proxy.WARNING@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CACHE]/goma/client/goma_stats.json", + "/path/to/tmp/" + ], + "env": { + "GOMA_DIR": "[CACHE]/goma/client", + "GOMA_TMP_DIR": "[CLEANUP]/goma", + "GOMA_USE_LOCAL": "False" + }, + "infra_step": true, + "name": "teardown goma.read goma_stats.json", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "[CACHE]/builder/src/script1.sh" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "darwin" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "name": "generator1" + }, + { + "cmd": [ + "[CACHE]/builder/src/myscript.sh", + "param1", + "param2" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "darwin" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "name": "mytest" + }, + { + "cmd": [ + "sudo", + "xcode-select", + "--reset" + ], + "infra_step": true, + "name": "reset XCode" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py new file mode 100644 index 0000000..7b4eb7e --- /dev/null +++ b/recipes/engine_v2/builder.py
@@ -0,0 +1,122 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Flutter Engine builder recipe. + +This recipe is used to build flavors of flutter engine identified by lists of +gn flags and ninja configs and targets. + + +The following are examples of valid configurations passed to builders using +this recipe in the builds property: + + { + "gn" : [ + "--ios", + "--runtime-mode", + "debug", + "--simulator", + "--no-lto" + ], + "ninja": { + "config": "ios_debug_sim", + "targets": ["ios_test_flutter"] + } + } +""" + +from contextlib import contextmanager + +from google.protobuf import struct_pb2 +from PB.recipes.flutter.engine import InputProperties +from PB.recipes.flutter.engine import EnvProperties +from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 + +DEPS = [ + 'flutter/build_util', + 'flutter/flutter_deps', + 'flutter/os_utils', + 'flutter/osx_sdk', + 'flutter/repo_util', + 'fuchsia/goma', + 'recipe_engine/context', + 'recipe_engine/file', + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/properties', + 'recipe_engine/step', +] + +PROPERTIES = InputProperties +ENV_PROPERTIES = EnvProperties + + +def Build(api, checkout): + """Builds a flavor identified as a set of gn and ninja configs.""" + ninja_tool = { + "ninja": api.build_util.build, + "autoninja": api.build_util.build_autoninja, + } + env, env_prefixes = api.repo_util.engine_environment(checkout) + deps = api.properties.get('dependencies', []) + api.flutter_deps.required_deps(env, env_prefixes, deps) + build = api.properties.get('build') + api.build_util.run_gn(build.get('gn'), checkout) + ninja = build.get('ninja') + ninja_tool[ninja.get('tool', 'ninja') + ](ninja.get('config'), checkout, ninja.get('targets')) + generators = build.get('generators', []) + # Get only local tests. + tests = [t for t in build.get('tests', []) if t.get('type') == 'local'] + with api.context(env=env, env_prefixes=env_prefixes, + cwd=checkout.join('flutter')): + # Generators must run from inside flutter folder. + for generator in generators: + api.step(generator.get('name'), [checkout.join(generator.get('script'))]) + # Run local tests in the builder to optimize resource usage. + for test in tests: + command = [test.get('language')] if test.get('language') else [] + # Ideally local tests should be completely hermetic and in theory we can run + # them in parallel using futures. I haven't found a flutter engine + # configuration with more than one local test but once we find it we + # should run the list of tests using parallelism. + # TODO(godofredoc): Optimize to run multiple local tests in parallel. + command.append(checkout.join(test.get('script'))) + command.extend(test.get('parameters', [])) + api.step(test.get('name'), command) + + +def RunSteps(api, properties, env_properties): + checkout = api.path['cache'].join('builder', 'src') + api.file.rmtree('Clobber build output', checkout.join('out')) + cache_root = api.path['cache'].join('builder') + api.file.ensure_directory('Ensure checkout cache', cache_root) + + # Enable long path support on Windows. + api.os_utils.enable_long_paths() + env = {} + env_prefixes = {} + + api.repo_util.engine_checkout(cache_root, env, env_prefixes) + if api.platform.is_mac: + with api.osx_sdk('ios'): + Build(api, checkout) + else: + Build(api, checkout) + + +def GenTests(api): + build = { + "gn": ["--ios"], "ninja": {"config": "ios_debug", "targets": []}, + "generators": [{"name": "generator1", + "script": "script1.sh"}], "tests": [{ + "name": "mytest", "script": "myscript.sh", + "parameters": ["param1", "param2"], "type": "local" + }] + } + yield api.test('basic', api.properties(build=build, goma_jobs="100")) + yield api.test( + 'mac', api.properties(build=build, goma_jobs="100"), + api.platform('mac', 64) + )
diff --git a/recipes/engine_v2/engine_v2.expected/basic.json b/recipes/engine_v2/engine_v2.expected/basic.json new file mode 100644 index 0000000..a9273e6 --- /dev/null +++ b/recipes/engine_v2/engine_v2.expected/basic.json
@@ -0,0 +1,326 @@ +[ + { + "cmd": [], + "name": "collect builds", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@{@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"status\": \"SUCCESS\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"value\": \"false\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"key\": \"cq_experimental\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ ], @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"builder\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"builder\": \"builder-subbuild1\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"bucket\": \"try\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"createdBy\": \"user:commit-bot@chromium.org\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"output\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"test_orchestration_inputs_hash\": \"abc\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"input\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"gerritChanges\": [@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"host\": \"chromium-review.googlesource.com\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"patchset\": \"7\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"change\": \"123456\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ ]@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"infra\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"swarming\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"priority\": 30@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"resultdb\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"invocation\": \"invocations/build:8945511751514863186\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@ \"id\": \"8945511751514863186\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-1-build.proto.json@}@@@", + "@@@STEP_LOG_END@fake-task-id-1-build.proto.json@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@{@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"status\": \"SUCCESS\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"value\": \"false\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"key\": \"cq_experimental\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ ], @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"builder\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"builder\": \"builder-subbuild2\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"bucket\": \"try\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"createdBy\": \"user:commit-bot@chromium.org\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"output\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"test_orchestration_inputs_hash\": \"abc\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"input\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"gerritChanges\": [@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"host\": \"chromium-review.googlesource.com\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"patchset\": \"7\", @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"change\": \"123456\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ ]@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"infra\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"swarming\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"priority\": 30@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"resultdb\": {@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"invocation\": \"invocations/build:8945511751514863187\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@ \"id\": \"8945511751514863187\"@@@", + "@@@STEP_LOG_LINE@fake-task-id-2-build.proto.json@}@@@", + "@@@STEP_LOG_END@fake-task-id-2-build.proto.json@@@" + ] + }, + { + "cmd": [], + "name": "collect builds.install infra/tools/luci/swarming", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin" + ], + "infra_step": true, + "name": "collect builds.install infra/tools/luci/swarming.ensure package directory", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin", + "-ensure-file", + "infra/tools/luci/swarming/${platform} swarming_module_pin", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "infra_step": true, + "name": "collect builds.install infra/tools/luci/swarming.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-swarming_module_\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/swarming/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin/swarming", + "collect", + "-server", + "https://example.swarmingserver.appspot.com", + "-task-summary-json", + "/path/to/tmp/json", + "-task-output-stdout", + "json", + "-output-dir", + "[CLEANUP]" + ], + "infra_step": true, + "name": "collect builds.collect", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"fake-task-id-1\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_code\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"builder-subbuild1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": \"COMPLETED\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"fake-task-id-1\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fake-task-id-2\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@", + "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@", + "@@@STEP_LOG_LINE@json.output@ \"exit_code\": 0, @@@", + "@@@STEP_LOG_LINE@json.output@ \"name\": \"builder-subbuild2\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"state\": \"COMPLETED\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"fake-task-id-2\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@STEP_LOG_LINE@task stdout+stderr: builder-subbuild1@hello world!@@@", + "@@@STEP_LOG_END@task stdout+stderr: builder-subbuild1@@@", + "@@@STEP_LOG_LINE@task stdout+stderr: builder-subbuild2@hello world!@@@", + "@@@STEP_LOG_END@task stdout+stderr: builder-subbuild2@@@", + "@@@STEP_LINK@task isolated outputs: builder-subbuild1@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@", + "@@@STEP_LINK@task isolated outputs: builder-subbuild2@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CLEANUP]/fake-task-id-1/build.proto.json", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "collect builds.read build.proto.json", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@build.proto.json@{@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"status\": \"SUCCESS\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"value\": \"false\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"key\": \"cq_experimental\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ ], @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder-subbuild1\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"try\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:commit-bot@chromium.org\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"output\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"test_orchestration_inputs_hash\": \"abc\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"gerritChanges\": [@@@", + "@@@STEP_LOG_LINE@build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium-review.googlesource.com\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"patchset\": \"7\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"change\": \"123456\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ ]@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:8945511751514863186\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"8945511751514863186\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@}@@@", + "@@@STEP_LOG_END@build.proto.json@@@" + ] + }, + { + "cmd": [ + "vpython", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CLEANUP]/fake-task-id-2/build.proto.json", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "collect builds.read build.proto.json (2)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@build.proto.json@{@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"status\": \"SUCCESS\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"tags\": [@@@", + "@@@STEP_LOG_LINE@build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"value\": \"false\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"key\": \"cq_experimental\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ ], @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder-subbuild2\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"try\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:commit-bot@chromium.org\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"output\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"test_orchestration_inputs_hash\": \"abc\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"gerritChanges\": [@@@", + "@@@STEP_LOG_LINE@build.proto.json@ {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"fuchsia\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium-review.googlesource.com\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"patchset\": \"7\", @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"change\": \"123456\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ ]@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:8945511751514863187\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }@@@", + "@@@STEP_LOG_LINE@build.proto.json@ }, @@@", + "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"8945511751514863187\"@@@", + "@@@STEP_LOG_LINE@build.proto.json@}@@@", + "@@@STEP_LOG_END@build.proto.json@@@" + ] + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipes/engine_v2/engine_v2.py b/recipes/engine_v2/engine_v2.py new file mode 100644 index 0000000..65cce2e --- /dev/null +++ b/recipes/engine_v2/engine_v2.py
@@ -0,0 +1,73 @@ +# Copyright 2021 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Orchestrator recipe that coordinates dependencies and build/test execution +# of sub-builds. +# +# This recipe is the main entry point for every builder(e.g. Windows Host +# Engine, Mac iOS Engine, etc). Once here this recipe will read the builder +# configuration from <engine_checkout>/ci/builders/<builder_name>, spawn +# subbuilds to compile/archive using engine_v2/builder.py wait for dependencies +# to be ready and then spawn subbuilds to run expensive tests using +# engine_v2/tester.py. + +from contextlib import contextmanager + +from PB.recipes.flutter.engine import InputProperties +from PB.recipes.flutter.engine import EnvProperties + +from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 +from google.protobuf import struct_pb2 + +DEPS = [ + 'flutter/shard_util_v2', + 'recipe_engine/buildbucket', + 'recipe_engine/properties', + 'recipe_engine/step', +] + +PROPERTIES = InputProperties +ENV_PROPERTIES = EnvProperties + + +def RunSteps(api, properties, env_properties): + builds = api.properties.get('builds') + tasks = api.shard_util_v2.schedule(builds) + with api.step.nest('collect builds') as presentation: + api.shard_util_v2.collect([build.build_id for build in tasks.values()], + presentation) + + +def GenTests(api): + try_subbuild1 = api.shard_util_v2.try_build_message( + build_id=8945511751514863186, + builder="builder-subbuild1", + output_props={"test_orchestration_inputs_hash": "abc"}, + status="SUCCESS", + ) + try_subbuild2 = api.shard_util_v2.try_build_message( + build_id=8945511751514863187, + builder="builder-subbuild2", + output_props={"test_orchestration_inputs_hash": "abc"}, + status="SUCCESS", + ) + builds = [{ + "name": "ios_debug", "gn": ["--ios"], + "ninja": {"config": "ios_debug", "targets": []}, + "generators": [{"name": "generator1", "script": "script1.sh"}] + }] + yield api.test( + 'basic', api.properties(builds=builds), + api.buildbucket.try_build( + project='proj', + builder='try-builder', + git_repo='https://github.com/repo/a', + revision='a' * 40, + build_number=123 + ), + api.shard_util_v2.child_led_steps( + builds=[try_subbuild1, try_subbuild2], + collect_step="collect builds", + ) + )