Reland: Simplify ruby and gems dependency.
This CL implements the following improvements:
* Removes complexity of gems installation using a single code path to
install and setup ruby.
* Fixes support for ruby arm64.
* Avoids the xcode ruby dependency.
* Generalizes gems path calculation to support multiple version of ruby
simultaneously.
Bug: https://github.com/flutter/flutter/issues/65888
Bug: https://github.com/flutter/flutter/issues/113231
Change-Id: I6a290401dc5bbc118a7b369f856a429ed6f67460
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/38088
Reviewed-by: Yusuf Mohsinally <mohsinally@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
(cherry picked from commit 189c50891c2418327d4eae0ddae23bb797a96419)
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/38160
Reviewed-by: Xilai Zhang <xilaizhang@google.com>
diff --git a/recipe_modules/flutter_deps/__init__.py b/recipe_modules/flutter_deps/__init__.py
index f1dfa6e..ebbb8ec 100644
--- a/recipe_modules/flutter_deps/__init__.py
+++ b/recipe_modules/flutter_deps/__init__.py
@@ -9,6 +9,7 @@
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
+ 'recipe_engine/raw_io',
'recipe_engine/step',
'recipe_engine/swarming',
]
diff --git a/recipe_modules/flutter_deps/api.py b/recipe_modules/flutter_deps/api.py
index 4ffd3d2..023cb06 100644
--- a/recipe_modules/flutter_deps/api.py
+++ b/recipe_modules/flutter_deps/api.py
@@ -88,7 +88,7 @@
'''.format(dependency)
raise ValueError(msg)
parsed_deps.append(dependency)
- if dependency in ['xcode', 'gems', 'swift', 'arm64ruby']:
+ if dependency in ['xcode', 'gems', 'swift']:
continue
dep_funct = available_deps.get(dependency)
if not dep_funct:
@@ -281,27 +281,25 @@
# For more, see CI section on https://docs.gradle.org/current/userguide/gradle_daemon.html#sec:disabling_the_daemon
env['GRADLE_OPTS'] = '-Dorg.gradle.daemon=false'
- def arm64ruby(self, env, env_prefixes, gem_dir):
+ def _install_ruby(self, env, env_prefixes, version=None):
"""Installs arm64 Ruby.
Context of arm64ruby:
go/benchmarks-on-platforms
https://github.com/flutter/flutter/issues/87508
"""
- version = 'version:311_3'
- with self.m.step.nest('Install arm64ruby'):
+ version = version or 'latest'
+ with self.m.step.nest('Install ruby'):
ruby_path = self.m.path['cache'].join('ruby')
ruby = self.m.cipd.EnsureFile()
- ruby.add_package('flutter/ruby/mac-arm64', version)
+ ruby.add_package('flutter/ruby/${platform}', version)
self.m.cipd.ensure(ruby_path, ruby)
paths = env_prefixes.get('PATH', [])
paths.insert(0, ruby_path.join('bin'))
env['RUBY_HOME'] = ruby_path.join('bin')
- env['GEM_HOME'] = gem_dir.join('ruby', '3.1.0')
- paths.append(gem_dir.join('ruby', '3.1.0', 'bin'))
env_prefixes['PATH'] = paths
- def gems(self, env, env_prefixes, gemfile_dir):
+ def gems(self, env, env_prefixes, gem_dir, version=None):
"""Installs mac gems.
Args:
@@ -310,41 +308,47 @@
gemfile_dir(Path): The path to the location of the repository gemfile.
"""
deps_list = self.m.properties.get('dependencies', [])
- deps = [d['dependency'] for d in deps_list]
- if 'gems' not in deps:
+ deps = {d['dependency']:d.get('version') for d in deps_list}
+ if 'gems' not in deps.keys():
# Noop if gems property is not set.
return
- gem_file = self.m.repo_util.sdk_checkout_path().join('flutter')
- gem_dir = self.m.path['start_dir'].join('gems')
- env['GEM_HOME'] = gem_dir
- if self.m.platform.arch == 'arm':
- self.arm64ruby(env, env_prefixes, gem_dir)
+ version = deps['gems']
+ gemfile_dir = gem_dir or self.m.repo_util.sdk_checkout_path().join('dev', 'ci', 'mac')
+ gem_destination = self.m.path['start_dir'].join('gems')
+ env['GEM_HOME'] = gem_destination
+ self._install_ruby(env, env_prefixes, version)
with self.m.step.nest('Install gems'):
- self.m.file.ensure_directory('mkdir gems', gem_dir)
- # Temporarily install bundler
- with self.m.context(cwd=gem_dir):
- self.m.step(
- 'install bundler',
- ['gem', 'install', 'bundler', '--install-dir', '.'],
- infra_step=True,
- )
+ self.m.file.ensure_directory('mkdir gems', gem_destination)
paths = env_prefixes.get('PATH', [])
temp_paths = copy.deepcopy(paths)
- temp_paths.append(gem_dir.join('bin'))
+ temp_paths.insert(0, gem_dir.join('bin'))
env_prefixes['PATH'] = temp_paths
with self.m.context(env=env, env_prefixes=env_prefixes, cwd=gemfile_dir):
self.m.step(
- 'set gems path',
- ['bundle', 'config', 'set', 'path', gem_dir],
+ 'Set gems path',
+ ['bundle', 'config', 'set', 'path', gem_destination],
+ infra_step=True,
+ )
+ opt_path = self.m.path['cache'].join('ruby', 'opt')
+ lib_path = self.m.path['cache'].join('ruby')
+ self.m.step(
+ 'Set ffi build flags',
+ ['bundle', 'config',
+ 'build.ffi', '--with-opt-dir=%s/gmp:%s' % (opt_path, lib_path)],
infra_step=True,
)
self.m.step('install gems', ['bundler', 'install'], infra_step=True)
+ # Find major/minor ruby version
+ ruby_version = self.m.step(
+ 'Ruby version', ['ruby', '-e', 'puts RUBY_PATCHLEVEL'],
+ stdout=self.m.raw_io.output_text(), ok_ret='any'
+ ).stdout.rstrip()
+ parts = ruby_version.split('.')
+ parts[-1] = '0'
+ ruby_version = '.'.join(parts)
+ paths.append(gem_destination.join('ruby', ruby_version, 'bin'))
# Update envs to the final destination.
- self.m.file.listdir('list bundle', gem_dir, recursive=True)
- if self.m.platform.arch != 'arm':
- env['GEM_HOME'] = gem_dir.join('ruby', '2.6.0')
- paths.append(gem_dir.join('ruby', '2.6.0', 'bin'))
- env_prefixes['PATH'] = paths
+ self.m.file.listdir('list bundle', gem_destination, recursive=True)
def firebase(self, env, env_prefixes, version='latest'):
"""Installs firebase binary.
diff --git a/recipe_modules/flutter_deps/examples/full.expected/with-arm64ruby.json b/recipe_modules/flutter_deps/examples/full.expected/with-arm64ruby.json
index 782134b..b925859 100644
--- a/recipe_modules/flutter_deps/examples/full.expected/with-arm64ruby.json
+++ b/recipe_modules/flutter_deps/examples/full.expected/with-arm64ruby.json
@@ -839,7 +839,7 @@
},
{
"cmd": [],
- "name": "Install arm64ruby"
+ "name": "Install ruby"
},
{
"cmd": [
@@ -848,7 +848,7 @@
"-root",
"[CACHE]/ruby",
"-ensure-file",
- "flutter/ruby/mac-arm64 version:311_3",
+ "flutter/ruby/${platform} latest",
"-max-threads",
"0",
"-json-output",
@@ -871,15 +871,15 @@
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
]
},
- "name": "Install arm64ruby.ensure_installed",
+ "name": "Install ruby.ensure_installed",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
"@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:311_3---\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"flutter/ruby/mac-arm64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"flutter/ruby/resolved-platform\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ ]@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
@@ -928,38 +928,6 @@
},
{
"cmd": [
- "gem",
- "install",
- "bundler",
- "--install-dir",
- "."
- ],
- "cwd": "[START_DIR]/gems",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter\\ sdk/bin",
- "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "Install gems.install bundler",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
"bundle",
"config",
"set",
@@ -969,7 +937,7 @@
"cwd": "[START_DIR]/dev/ci/mac",
"env": {
"DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GEM_HOME": "[START_DIR]/gems/ruby/3.1.0",
+ "GEM_HOME": "[START_DIR]/gems",
"GIT_BRANCH": "",
"LUCI_BRANCH": "",
"LUCI_CI": "True",
@@ -982,17 +950,53 @@
},
"env_prefixes": {
"PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
"[CACHE]/ruby/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
- "[START_DIR]/gems/ruby/3.1.0/bin",
- "[START_DIR]/gems/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
]
},
"infra_step": true,
- "name": "Install gems.set gems path",
+ "name": "Install gems.Set gems path",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bundle",
+ "config",
+ "build.ffi",
+ "--with-opt-dir=[CACHE]/ruby/opt/gmp:[CACHE]/ruby"
+ ],
+ "cwd": "[START_DIR]/dev/ci/mac",
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GEM_HOME": "[START_DIR]/gems",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "RUBY_HOME": "[CACHE]/ruby/bin",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
+ "[CACHE]/ruby/bin",
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Install gems.Set ffi build flags",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -1005,7 +1009,7 @@
"cwd": "[START_DIR]/dev/ci/mac",
"env": {
"DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GEM_HOME": "[START_DIR]/gems/ruby/3.1.0",
+ "GEM_HOME": "[START_DIR]/gems",
"GIT_BRANCH": "",
"LUCI_BRANCH": "",
"LUCI_CI": "True",
@@ -1018,11 +1022,10 @@
},
"env_prefixes": {
"PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
"[CACHE]/ruby/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
- "[START_DIR]/gems/ruby/3.1.0/bin",
- "[START_DIR]/gems/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
]
@@ -1035,6 +1038,34 @@
},
{
"cmd": [
+ "ruby",
+ "-e",
+ "puts RUBY_PATCHLEVEL"
+ ],
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "name": "Install gems.Ruby version",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
"vpython3",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
diff --git a/recipe_modules/flutter_deps/examples/full.expected/with-gems.json b/recipe_modules/flutter_deps/examples/full.expected/with-gems.json
index d9c7054..8c23b12 100644
--- a/recipe_modules/flutter_deps/examples/full.expected/with-gems.json
+++ b/recipe_modules/flutter_deps/examples/full.expected/with-gems.json
@@ -839,6 +839,56 @@
},
{
"cmd": [],
+ "name": "Install ruby"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/ruby",
+ "-ensure-file",
+ "flutter/ruby/${platform} v3.1.3",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "name": "Install ruby.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-v3.1.3----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"flutter/ruby/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
"name": "Install gems"
},
{
@@ -878,38 +928,6 @@
},
{
"cmd": [
- "gem",
- "install",
- "bundler",
- "--install-dir",
- "."
- ],
- "cwd": "[START_DIR]/gems",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter\\ sdk/bin",
- "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "Install gems.install bundler",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
"bundle",
"config",
"set",
@@ -927,19 +945,58 @@
"OS": "linux",
"PUB_CACHE": "[START_DIR]/.pub-cache",
"REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "RUBY_HOME": "[CACHE]/ruby/bin",
"SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
},
"env_prefixes": {
"PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
+ "[CACHE]/ruby/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
- "[START_DIR]/gems/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
]
},
"infra_step": true,
- "name": "Install gems.set gems path",
+ "name": "Install gems.Set gems path",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "bundle",
+ "config",
+ "build.ffi",
+ "--with-opt-dir=[CACHE]/ruby/opt/gmp:[CACHE]/ruby"
+ ],
+ "cwd": "[START_DIR]/dev/ci/mac",
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GEM_HOME": "[START_DIR]/gems",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "RUBY_HOME": "[CACHE]/ruby/bin",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
+ "[CACHE]/ruby/bin",
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Install gems.Set ffi build flags",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
@@ -960,13 +1017,15 @@
"OS": "linux",
"PUB_CACHE": "[START_DIR]/.pub-cache",
"REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "RUBY_HOME": "[CACHE]/ruby/bin",
"SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
},
"env_prefixes": {
"PATH": [
+ "[START_DIR]/dev/ci/mac/bin",
+ "[CACHE]/ruby/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin",
- "[START_DIR]/gems/bin",
"[START_DIR]/flutter\\ sdk/bin",
"[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
]
@@ -979,6 +1038,34 @@
},
{
"cmd": [
+ "ruby",
+ "-e",
+ "puts RUBY_PATCHLEVEL"
+ ],
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter\\ sdk"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter\\ sdk/bin",
+ "[START_DIR]/flutter\\ sdk/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "name": "Install gems.Ruby version",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
"vpython3",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
diff --git a/recipe_modules/flutter_deps/examples/full.py b/recipe_modules/flutter_deps/examples/full.py
index 12f3bf7..422ae00 100644
--- a/recipe_modules/flutter_deps/examples/full.py
+++ b/recipe_modules/flutter_deps/examples/full.py
@@ -90,7 +90,7 @@
api.repo_util.flutter_environment_data(checkout_path),
)
yield api.test(
- 'with-gems', api.properties(dependencies=[{"dependency": "gems"}]),
+ 'with-gems', api.properties(dependencies=[{"dependency": "gems", "version": "v3.1.3"}]),
api.repo_util.flutter_environment_data(checkout_path),
)
