Collect clang crash dumps for rbe build
Follow up of https://flutter-review.googlesource.com/c/recipes/+/53421 to support RBE build.
Change-Id: I118444703b9dfa86b033627b11289e00b4487017
Bug: https://github.com/flutter/flutter/issues/140458
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/55320
Commit-Queue: Keyong Han <keyonghan@google.com>
Reviewed-by: Ricardo Amador <ricardoamador@google.com>
diff --git a/recipe_modules/build_util/api.py b/recipe_modules/build_util/api.py
index a8603b6..7bf9fdc 100644
--- a/recipe_modules/build_util/api.py
+++ b/recipe_modules/build_util/api.py
@@ -65,7 +65,7 @@
j_value = min(j_value, 800)
return 200 if self._test_data.enabled else j_value
- def _build_rbe(self, config, checkout_path, targets, tool, rbe_working_path):
+ def _build_rbe(self, config, checkout_path, targets, tool, rbe_working_path, env):
"""Builds using ninja and rbe.
Args:
@@ -85,8 +85,12 @@
collect_rbe_logs_latency=self.m.properties.get(
'collect_rbe_logs_latency',
COLLECT_RBE_LOGS_LATENCY_SECS)), self.m.depot_tools.on_path():
- name = 'build %s' % ' '.join([config] + list(targets))
- self.m.step(name, ninja_args)
+ try:
+ name = 'build %s' % ' '.join([config] + list(targets))
+ self.m.step(name, ninja_args)
+ except self.m.step.StepFailure:
+ self._upload_crash_reproducer(env)
+ raise
def _build_goma(self, config, checkout_path, targets, tool, env):
"""Builds using ninja and goma.
@@ -166,7 +170,7 @@
ninja_path = checkout_path.join('flutter', 'third_party', 'ninja', 'ninja')
if self.use_rbe:
self._build_rbe(
- config, checkout_path, targets, ninja_path, rbe_working_path
+ config, checkout_path, targets, ninja_path, rbe_working_path, env
)
else:
if self.use_goma:
diff --git a/recipe_modules/build_util/examples/full.expected/basic.json b/recipe_modules/build_util/examples/full.expected/basic.json
index 9875f79..d358677 100644
--- a/recipe_modules/build_util/examples/full.expected/basic.json
+++ b/recipe_modules/build_util/examples/full.expected/basic.json
@@ -814,7 +814,7 @@
"5",
"-C",
"[START_DIR]/out/release",
- "mytarget"
+ "rbe_target"
],
"env": {
"RBE_cache_dir": "[CACHE]/rbe/deps",
@@ -839,7 +839,7 @@
"RECIPE_REPO[depot_tools]"
]
},
- "name": "build release mytarget (2)"
+ "name": "build release rbe_target"
},
{
"cmd": [],
diff --git a/recipe_modules/build_util/examples/full.expected/mac.json b/recipe_modules/build_util/examples/full.expected/mac.json
index 94b63fd..ea5dd63 100644
--- a/recipe_modules/build_util/examples/full.expected/mac.json
+++ b/recipe_modules/build_util/examples/full.expected/mac.json
@@ -814,7 +814,7 @@
"5",
"-C",
"[START_DIR]/out/release",
- "mytarget"
+ "rbe_target"
],
"env": {
"RBE_cache_dir": "[CACHE]/rbe/deps",
@@ -839,7 +839,138 @@
"RECIPE_REPO[depot_tools]"
]
},
- "name": "build release mytarget (2)"
+ "name": "build release rbe_target",
+ "~followup_annotations": [
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "upload crash reproducer"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]",
+ "*.sh"
+ ],
+ "env": {
+ "RBE_cache_dir": "[CACHE]/rbe/deps",
+ "RBE_deps_cache_max_mb": "512",
+ "RBE_enable_deps_cache": "true",
+ "RBE_instance": "fake_rbe_instance",
+ "RBE_log_dir": "[CLEANUP]/rbe",
+ "RBE_log_format": "reducedtext",
+ "RBE_output_dir": "[CLEANUP]/rbe",
+ "RBE_proxy_log_dir": "[CLEANUP]/rbe",
+ "RBE_server_address": "unix://[CLEANUP]/rbe/reproxy.sock",
+ "RBE_service": "remotebuildexecution.googleapis.com:443",
+ "RBE_socket_path": "[CLEANUP]/rbe/reproxy.sock",
+ "RBE_use_application_default_credentials": "false",
+ "RBE_use_gce_credentials": "true"
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "upload crash reproducer.find reproducers",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/[START_DIR]/foo.sh@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]",
+ "foo.*"
+ ],
+ "env": {
+ "RBE_cache_dir": "[CACHE]/rbe/deps",
+ "RBE_deps_cache_max_mb": "512",
+ "RBE_enable_deps_cache": "true",
+ "RBE_instance": "fake_rbe_instance",
+ "RBE_log_dir": "[CLEANUP]/rbe",
+ "RBE_log_format": "reducedtext",
+ "RBE_output_dir": "[CLEANUP]/rbe",
+ "RBE_proxy_log_dir": "[CLEANUP]/rbe",
+ "RBE_server_address": "unix://[CLEANUP]/rbe/reproxy.sock",
+ "RBE_service": "remotebuildexecution.googleapis.com:443",
+ "RBE_socket_path": "[CLEANUP]/rbe/reproxy.sock",
+ "RBE_use_application_default_credentials": "false",
+ "RBE_use_gce_credentials": "true"
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "upload crash reproducer.find foo files",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/[START_DIR]/foo.sh@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/[START_DIR]/foo.sh",
+ "[START_DIR]"
+ ],
+ "env": {
+ "RBE_cache_dir": "[CACHE]/rbe/deps",
+ "RBE_deps_cache_max_mb": "512",
+ "RBE_enable_deps_cache": "true",
+ "RBE_instance": "fake_rbe_instance",
+ "RBE_log_dir": "[CLEANUP]/rbe",
+ "RBE_log_format": "reducedtext",
+ "RBE_output_dir": "[CLEANUP]/rbe",
+ "RBE_proxy_log_dir": "[CLEANUP]/rbe",
+ "RBE_server_address": "unix://[CLEANUP]/rbe/reproxy.sock",
+ "RBE_service": "remotebuildexecution.googleapis.com:443",
+ "RBE_socket_path": "[CLEANUP]/rbe/reproxy.sock",
+ "RBE_use_application_default_credentials": "false",
+ "RBE_use_gce_credentials": "true"
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "upload crash reproducer.Copy crash reproduce file [START_DIR]/[START_DIR]/foo.sh",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
},
{
"cmd": [],
@@ -1144,6 +1275,10 @@
]
},
{
+ "failure": {
+ "failure": {},
+ "humanReason": "Step('build release rbe_target') (retcode: 1)"
+ },
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipe_modules/build_util/examples/full.py b/recipe_modules/build_util/examples/full.py
index 0dd1f12..ee23245 100644
--- a/recipe_modules/build_util/examples/full.py
+++ b/recipe_modules/build_util/examples/full.py
@@ -37,7 +37,7 @@
api.build_util.build(
'release',
checkout,
- ['mytarget'],
+ ['rbe_target'],
{
'CLANG_CRASH_DIAGNOSTICS_DIR': api.path['start_dir'],
'FLUTTER_LOGS_DIR': api.path['start_dir']
@@ -68,4 +68,13 @@
),
status='FAILURE',
)
- yield api.test('mac', api.properties(no_lto=True), api.platform('mac', 64))
+ yield api.test(
+ 'mac',
+ api.properties(no_lto=True),
+ api.platform('mac', 64),
+ api.step_data(
+ 'build release rbe_target',
+ retcode=1,
+ ),
+ status='FAILURE',
+ )
