Clean all git repositories in the checkout path when building a release branch
This is part of an attempt to find a faster way to ensure a sufficiently clean source tree for release branches.
Release branch builds are currently doing a clobber of the bot caches (see https://flutter.googlesource.com/recipes/+/ee6c6a5cec8f3ebbd0a5661ab19a03831a5dc465). It should be possible to avoid that by removing any files in the source repositories that are not tracked by Git.
For now, this PR keeps the clobber while we continue investigating any other state on the bots that may need to be cleaned.
See https://github.com/flutter/flutter/issues/169887
Change-Id: I8ef7865bf5273fcc89513bd056c7451bcf973e59
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/66103
Reviewed-by: Matan Lurey <matanl@google.com>
Commit-Queue: Jason Simmons <jsimmons@google.com>
diff --git a/recipe_modules/repo_util/api.py b/recipe_modules/repo_util/api.py
index 4cfac43..5db0549 100644
--- a/recipe_modules/repo_util/api.py
+++ b/recipe_modules/repo_util/api.py
@@ -127,6 +127,12 @@
# automatically deleted for example when try bots checkout an old hash.
clobber = True
+ # Try doing a git clean of every repository.
+ # This is an experiment that may serve as a faster alternative to
+ # clobbering the entire cache for release branch builds.
+ with self.m.context(cwd=checkout_path):
+ self.m.gclient('git clean all repos', ['recurse', 'git', 'clean', '-fdx'])
+
engine_repo = 'flutter' if self.is_fusion() else 'engine'
git_url = REPOS[engine_repo]
git_id = self.m.buildbucket.gitiles_commit.id
diff --git a/recipe_modules/repo_util/examples/full.expected/force_get_candidate_branch.json b/recipe_modules/repo_util/examples/full.expected/force_get_candidate_branch.json
index 13b8b25..0ebbdbe 100644
--- a/recipe_modules/repo_util/examples/full.expected/force_get_candidate_branch.json
+++ b/recipe_modules/repo_util/examples/full.expected/force_get_candidate_branch.json
@@ -53,7 +53,7 @@
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^",
" File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/examples/full.py\", line 28, in RunSteps",
" api.repo_util.release_candidate_branch(flutter_checkout_path)",
- " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 618, in release_candidate_branch",
+ " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 624, in release_candidate_branch",
" raise ValueError('Not a release candidate branch: %s' % candidate_branch)",
"ValueError: Not a release candidate branch: refs/pull/1/head"
]
diff --git a/recipe_modules/repo_util/examples/full.expected/mac.json b/recipe_modules/repo_util/examples/full.expected/mac.json
index c298036..6994b5e 100644
--- a/recipe_modules/repo_util/examples/full.expected/mac.json
+++ b/recipe_modules/repo_util/examples/full.expected/mac.json
@@ -768,6 +768,25 @@
"name": "ensure directory"
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[START_DIR]/engine",
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipe_modules/repo_util/examples/full.expected/mac_release_candidate.json b/recipe_modules/repo_util/examples/full.expected/mac_release_candidate.json
index fe01cc0..1dd83ca 100644
--- a/recipe_modules/repo_util/examples/full.expected/mac_release_candidate.json
+++ b/recipe_modules/repo_util/examples/full.expected/mac_release_candidate.json
@@ -808,6 +808,25 @@
"name": "ensure directory"
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[START_DIR]/engine",
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipe_modules/repo_util/examples/full.expected/mac_release_candidate_sha_mismatch.json b/recipe_modules/repo_util/examples/full.expected/mac_release_candidate_sha_mismatch.json
index d6fc49e..985aebf 100644
--- a/recipe_modules/repo_util/examples/full.expected/mac_release_candidate_sha_mismatch.json
+++ b/recipe_modules/repo_util/examples/full.expected/mac_release_candidate_sha_mismatch.json
@@ -788,6 +788,25 @@
"name": "ensure directory"
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[START_DIR]/engine",
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipe_modules/repo_util/examples/full.expected/monorepo_wrong_host.json b/recipe_modules/repo_util/examples/full.expected/monorepo_wrong_host.json
index 72b9db1..fd9532e 100644
--- a/recipe_modules/repo_util/examples/full.expected/monorepo_wrong_host.json
+++ b/recipe_modules/repo_util/examples/full.expected/monorepo_wrong_host.json
@@ -1044,7 +1044,7 @@
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^",
" File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/examples/full.py\", line 54, in RunSteps",
" api.repo_util.monorepo_checkout(checkout_path, {}, {})",
- " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 221, in monorepo_checkout",
+ " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 227, in monorepo_checkout",
" raise ValueError(",
"ValueError: Input reference is not on dart.googlesource.com/monorepo"
]
diff --git a/recipe_modules/repo_util/examples/full.expected/win.json b/recipe_modules/repo_util/examples/full.expected/win.json
index 24418a1..f985baa 100644
--- a/recipe_modules/repo_util/examples/full.expected/win.json
+++ b/recipe_modules/repo_util/examples/full.expected/win.json
@@ -767,6 +767,25 @@
"name": "ensure directory"
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]\\gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[START_DIR]\\engine",
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json b/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json
index 3fb0f42..17bdd01 100644
--- a/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json
+++ b/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json
@@ -21,7 +21,7 @@
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^",
" File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/examples/unsupported.py\", line 13, in RunSteps",
" api.repo_util.checkout('unsupported_repo', repo_dir)",
- " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 287, in checkout",
+ " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 293, in checkout",
" raise ValueError('Unsupported repo: %s' % name)",
"ValueError: Unsupported repo: unsupported_repo"
]
diff --git a/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json b/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json
index 3474bae..6440ce4 100644
--- a/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json
+++ b/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json
@@ -216,6 +216,40 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[CACHE]/builder",
+ "env": {
+ "DEPOT_TOOLS_REPORT_BUILD": "dart-internal/flutter/builder/8945511751514863184"
+ },
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart-internal:flutter"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipes/engine_v2/builder.expected/test_if_skip_presubmit.json b/recipes/engine_v2/builder.expected/test_if_skip_presubmit.json
index 320db0b..8cc0c2d 100644
--- a/recipes/engine_v2/builder.expected/test_if_skip_presubmit.json
+++ b/recipes/engine_v2/builder.expected/test_if_skip_presubmit.json
@@ -165,6 +165,40 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[CACHE]/builder",
+ "env": {
+ "DEPOT_TOOLS_REPORT_BUILD": "flutter/prod/linux-host/8945511751514863184"
+ },
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "flutter:prod"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
diff --git a/recipes/engine_v2/engine_v2.expected/basic_mac_dart_internal.json b/recipes/engine_v2/engine_v2.expected/basic_mac_dart_internal.json
index 814763b..e4ba60c 100644
--- a/recipes/engine_v2/engine_v2.expected/basic_mac_dart_internal.json
+++ b/recipes/engine_v2/engine_v2.expected/basic_mac_dart_internal.json
@@ -451,6 +451,40 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "recurse",
+ "git",
+ "clean",
+ "-fdx"
+ ],
+ "cwd": "[CACHE]/builder",
+ "env": {
+ "DEPOT_TOOLS_REPORT_BUILD": "dart-internal/flutter/prod-builder/8945511751514863184"
+ },
+ "env_suffixes": {
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart-internal:flutter"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gclient git clean all repos"
+ },
+ {
"cmd": [],
"name": "Checkout source code"
},
