[fuchsia-ctl] Fix checkout path The checkout dir was broken due to the way that api.git calculates it. See https://ci.chromium.org/p/flutter/builders/try/fuchsia_ctl/3. This change fixes the issue by specifying the checkout path explicitly. See https://chromium-swarm.appspot.com/task?id=4c514fb5cd68b110 Bug: https://github.com/flutter/flutter/issues/56138 Change-Id: Ia26b6ec9fcae655b169016a14927257fc95be77a Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/3045 Reviewed-by: Godofredo Contreras <godofredoc@google.com> Commit-Queue: Tong Wu <wutong@google.com>
diff --git a/recipe_modules/repo_util/api.py b/recipe_modules/repo_util/api.py index 0b1e291..c459f62 100644 --- a/recipe_modules/repo_util/api.py +++ b/recipe_modules/repo_util/api.py
@@ -2,9 +2,16 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -FLUTTER_REPO = 'https://chromium.googlesource.com/external/github.com/flutter/flutter' -ENGINE_REPO = 'https://chromium.googlesource.com/external/github.com/flutter/engine' -COCOON_REPO = 'https://chromium.googlesource.com/external/github.com/flutter/cocoon' +REPOS = { + 'flutter': + 'https://chromium.googlesource.com/external/github.com/flutter/flutter', + 'engine': + 'https://chromium.googlesource.com/external/github.com/flutter/engine', + 'cocoon': + 'https://chromium.googlesource.com/external/github.com/flutter/cocoon', + 'packages': + 'https://github.com/flutter/packages', +} from recipe_engine import recipe_api @@ -12,43 +19,47 @@ class RepoUtilApi(recipe_api.RecipeApi): """Provides utilities to work with flutter repos.""" - def checkout_flutter(self, checkout_path, url=None, ref=None): - git_url = url or FLUTTER_REPO - git_ref = ref or self.m.buildbucket.gitiles_commit.ref - return self.m.git.checkout( - git_url, ref=git_ref, recursive=True, set_got_revision=True, tags=True) + def checkout(self, name, checkout_path, url=None, ref=None): + """Checks out a repo and returns sha1 of checked out revision. - def checkout_engine(self, checkout_path, url=None, ref=None): - git_url = url or ENGINE_REPO - git_ref = ref or self.m.buildbucket.gitiles_commit.ref - return self.m.git.checkout( - git_url, ref=git_ref, recursive=True, set_got_revision=True, tags=True) + The supproted repository names and their urls are defined in the global + REPOS variable. - def checkout_cocoon(self, checkout_path, url=None, ref=None): - git_url = url or COCOON_REPO + Args: + name (str): name of the supported repository. + checkout_path (Path): directory to clone into. + url (str): optional url overwrite of the remote repo. + ref (str): optional ref overwrite to fetch and check out. + """ + if name not in REPOS: + raise ValueError('Unsupported repo: %s' % name) + + git_url = url or REPOS[name] git_ref = ref or self.m.buildbucket.gitiles_commit.ref return self.m.git.checkout( - git_url, ref=git_ref, recursive=True, set_got_revision=True, tags=True) + git_url, + dir_path=checkout_path, + ref=git_ref, + recursive=True, + set_got_revision=True, + tags=True) def flutter_environment(self, checkout_path): + """Returns env and env_prefixes of an flutter/dart command environment.""" dart_bin = checkout_path.join('bin', 'cache', 'dart-sdk', 'bin') flutter_bin = checkout_path.join('bin') - path_prefixes = [ - flutter_bin, - dart_bin, - ] # Fail if dart and flutter bin folders do not exist. if not (self.m.path.exists(dart_bin) and self.m.path.exists(flutter_bin)): msg = ('dart or flutter bin folders do not exist,' 'did you forget to checkout flutter repo?') self.m.python.failing_step('Flutter Environment', msg) - env_prefixes = {'PATH': path_prefixes} - pub_cache = checkout_path.join('.pub-cache') + env = { # Setup our own pub_cache to not affect other slaves on this machine, # and so that the pre-populated pub cache is contained in the package. - 'PUB_CACHE': pub_cache, + 'PUB_CACHE': checkout_path.join('.pub-cache'), # Windows Packaging script assumes this is set. 'DEPOT_TOOLS': str(self.m.depot_tools.root), } + env_prefixes = {'PATH': [flutter_bin, dart_bin]} return env, env_prefixes
diff --git a/recipe_modules/repo_util/examples/full.expected/basic.json b/recipe_modules/repo_util/examples/full.expected/basic.json index eaa286d..38a28e4 100644 --- a/recipe_modules/repo_util/examples/full.expected/basic.json +++ b/recipe_modules/repo_util/examples/full.expected/basic.json
@@ -264,6 +264,94 @@ "name": "submodule update (3)" }, { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/packages", + "--url", + "https://github.com/flutter/packages" + ], + "name": "git setup (4)" + }, + { + "cmd": [ + "git", + "fetch", + "origin", + "master", + "--recurse-submodules", + "--progress", + "--tags" + ], + "cwd": "[START_DIR]/packages", + "env": { + "PATH": "RECIPE_REPO[depot_tools]:<PATH>" + }, + "infra_step": true, + "name": "git fetch (4)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "git checkout (4)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "read revision (4)", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "git clean (4)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "submodule sync (4)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "submodule update (4)" + }, + { "name": "$result" } ] \ No newline at end of file
diff --git a/recipe_modules/repo_util/examples/full.expected/failed_flutter_environment.json b/recipe_modules/repo_util/examples/full.expected/failed_flutter_environment.json index 2c2e648..3595182 100644 --- a/recipe_modules/repo_util/examples/full.expected/failed_flutter_environment.json +++ b/recipe_modules/repo_util/examples/full.expected/failed_flutter_environment.json
@@ -267,6 +267,94 @@ "cmd": [ "python", "-u", + "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", + "--path", + "[START_DIR]/packages", + "--url", + "https://github.com/flutter/packages" + ], + "name": "git setup (4)" + }, + { + "cmd": [ + "git", + "fetch", + "origin", + "master", + "--recurse-submodules", + "--progress", + "--tags" + ], + "cwd": "[START_DIR]/packages", + "env": { + "PATH": "RECIPE_REPO[depot_tools]:<PATH>" + }, + "infra_step": true, + "name": "git fetch (4)" + }, + { + "cmd": [ + "git", + "checkout", + "-f", + "FETCH_HEAD" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "git checkout (4)" + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "read revision (4)", + "~followup_annotations": [ + "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@" + ] + }, + { + "cmd": [ + "git", + "clean", + "-f", + "-d", + "-x" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "git clean (4)" + }, + { + "cmd": [ + "git", + "submodule", + "sync" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "submodule sync (4)" + }, + { + "cmd": [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], + "cwd": "[START_DIR]/packages", + "infra_step": true, + "name": "submodule update (4)" + }, + { + "cmd": [ + "python", + "-u", "import sys; sys.exit(1)" ], "name": "Flutter Environment",
diff --git a/recipe_modules/repo_util/examples/full.py b/recipe_modules/repo_util/examples/full.py index b3870fb..d144e7d 100644 --- a/recipe_modules/repo_util/examples/full.py +++ b/recipe_modules/repo_util/examples/full.py
@@ -11,11 +11,12 @@ def RunSteps(api): - checkout_path = api.path['checkout'] - api.repo_util.checkout_flutter(checkout_path) - api.repo_util.checkout_engine(checkout_path) - api.repo_util.checkout_cocoon(checkout_path) - env, env_paths = api.repo_util.flutter_environment(checkout_path) + flutter_checkout_path = api.path['start_dir'].join('flutter') + api.repo_util.checkout('flutter', flutter_checkout_path) + api.repo_util.checkout('engine', api.path['start_dir'].join('engine')) + api.repo_util.checkout('cocoon', api.path['start_dir'].join('cocoon')) + api.repo_util.checkout('packages', api.path['start_dir'].join('packages')) + env, env_paths = api.repo_util.flutter_environment(flutter_checkout_path) def GenTests(api):
diff --git a/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json b/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json new file mode 100644 index 0000000..8a6001d --- /dev/null +++ b/recipe_modules/repo_util/examples/unsupported.expected/unsupported.json
@@ -0,0 +1,33 @@ +[ + { + "cmd": [], + "name": "RECIPE CRASH (Uncaught exception)", + "~followup_annotations": [ + "@@@STEP_EXCEPTION@@@", + "The recipe has crashed at point 'Uncaught exception'!", + "", + "Traceback (most recent call last):", + " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", in run_steps", + " raw_result = recipe_obj.run_steps(api, engine)", + " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", in run_steps", + " properties_def, api=api)", + " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in invoke_with_properties", + " arg_names, **additional_args)", + " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in _invoke_with_properties", + " return callable_obj(*props, **additional_args)", + " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/examples/unsupported.py\", line 15, in RunSteps", + " api.path['start_dir'].join('unsupported_repo'))", + " File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in _inner", + " return func(*a, **kw)", + " File \"RECIPE_REPO[flutter]/recipe_modules/repo_util/api.py\", line 35, in checkout", + " raise ValueError('Unsupported repo: %s' % name)", + "ValueError: Unsupported repo: unsupported_repo" + ] + }, + { + "failure": { + "humanReason": "Uncaught Exception: ValueError('Unsupported repo: unsupported_repo',)" + }, + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/repo_util/examples/unsupported.py b/recipe_modules/repo_util/examples/unsupported.py new file mode 100644 index 0000000..f812142 --- /dev/null +++ b/recipe_modules/repo_util/examples/unsupported.py
@@ -0,0 +1,22 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import DoesNotRun, Filter, StatusFailure + +DEPS = [ + 'flutter/repo_util', + 'recipe_engine/path', +] + + +def RunSteps(api): + api.repo_util.checkout('unsupported_repo', + api.path['start_dir'].join('unsupported_repo')) + + +def GenTests(api): + yield api.test( + 'unsupported', + api.expect_exception('ValueError'), + )
diff --git a/recipes/fuchsia/fuchsia.py b/recipes/fuchsia/fuchsia.py index 4038a74..b43f9ea 100644 --- a/recipes/fuchsia/fuchsia.py +++ b/recipes/fuchsia/fuchsia.py
@@ -16,9 +16,13 @@ def RunSteps(api): - checkout_path = api.path['checkout'] - api.repo_util.checkout_flutter(checkout_path, api.properties.get('git_url'), - api.properties.get('git_ref')) + checkout_path = api.path['start_dir'].join('flutter') + api.repo_util.checkout( + 'flutter', + checkout_path, + api.properties.get('git_url'), + api.properties.get('git_ref'), + ) env, env_prefixes = api.repo_util.flutter_environment(checkout_path) with api.context(env=env, env_prefixes=env_prefixes, cwd=checkout_path): metadata = api.fuchsia_util.run_test(checkout_path)
diff --git a/recipes/fuchsia_ctl.expected/demo.json b/recipes/fuchsia_ctl.expected/demo.json index 96cc09e..cb524e4 100644 --- a/recipes/fuchsia_ctl.expected/demo.json +++ b/recipes/fuchsia_ctl.expected/demo.json
@@ -5,7 +5,7 @@ "-u", "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py", "--path", - "[START_DIR]/repo", + "[START_DIR]/packages", "--url", "https://abc.com/repo" ], @@ -21,7 +21,7 @@ "--progress", "--tags" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "env": { "PATH": "RECIPE_REPO[depot_tools]:<PATH>" }, @@ -35,7 +35,7 @@ "-f", "FETCH_HEAD" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "infra_step": true, "name": "git checkout" }, @@ -45,7 +45,7 @@ "rev-parse", "HEAD" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "infra_step": true, "name": "read revision", "~followup_annotations": [ @@ -61,7 +61,7 @@ "-d", "-x" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "infra_step": true, "name": "git clean" }, @@ -71,7 +71,7 @@ "submodule", "sync" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "infra_step": true, "name": "submodule sync" }, @@ -83,7 +83,7 @@ "--init", "--recursive" ], - "cwd": "[START_DIR]/repo", + "cwd": "[START_DIR]/packages", "infra_step": true, "name": "submodule update" }, @@ -91,7 +91,7 @@ "cmd": [ "tool/build.sh" ], - "cwd": "[START_DIR]/packages/fuchsia_ctl", + "cwd": "[START_DIR]/packages/packages/fuchsia_ctl", "name": "run tool/build.sh" }, { @@ -99,7 +99,7 @@ "cipd", "pkg-build", "-in", - "[START_DIR]/packages/fuchsia_ctl/build", + "[START_DIR]/packages/packages/fuchsia_ctl/build", "-name", "flutter/fuchsia_ctl/${platform}", "-out",
diff --git a/recipes/fuchsia_ctl.py b/recipes/fuchsia_ctl.py index b674c55..1e08dd6 100644 --- a/recipes/fuchsia_ctl.py +++ b/recipes/fuchsia_ctl.py
@@ -17,6 +17,7 @@ 'recipe_engine/properties', 'recipe_engine/step', 'recipe_engine/swarming', + 'repo_util', 'yaml', ] @@ -24,24 +25,16 @@ # This recipe builds the fuchsia_ctl CIPD package and tests it against the Linux # Fuchsia builder. def RunSteps(api): - # Checkout the flutter/packages repository. - packages_git_url = 'https://github.com/flutter/packages/' - if 'git_url' in api.properties: - packages_git_url = api.properties['git_url'] - - packages_git_ref = 'master' - if 'git_ref' in api.properties: - packages_git_ref = api.properties['git_ref'] - - packages_git_hash = api.git.checkout( - packages_git_url, - ref=packages_git_ref, - recursive=True, - set_got_revision=True, - tags=True) + packages_dir = api.path['start_dir'].join('packages') + packages_git_rev = api.repo_util.checkout( + 'packages', + packages_dir, + api.properties['git_url'], + api.properties['git_ref'], + ) # Build and uploads a new version of the fuchsia_ctl CIPD package. - fuchsia_ctl_path = api.path['start_dir'].join('packages', 'fuchsia_ctl') + fuchsia_ctl_path = packages_dir.join('packages', 'fuchsia_ctl') with api.context(cwd=fuchsia_ctl_path): api.step('run tool/build.sh', cmd=['tool/build.sh'])