blob: eb820bd3e9ff6186912a01eb4502eff018bc17dd [file] [log] [blame]
# Copyright 2020 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import attr
from recipe_engine import recipe_api
from recipe_engine.config_types import Path
SDK_GCS_BUCKET = 'fuchsia'
@attr.s
class ImageFilePaths(object):
"""Required files from fuchsia image to start FEMU."""
# Recipe API, required
_api = attr.ib(type=recipe_api.RecipeApi)
# Files from fuchsia image
build_args = attr.ib(type=Path, default=None)
kernel_file = attr.ib(type=Path, default=None)
system_fvm = attr.ib(type=Path, default=None)
zircona = attr.ib(type=Path, default=None)
def _exists(self, p):
return p and self._api.path.exists(p)
def _exist(self):
return all([
self._exists(self.build_args),
self._exists(self.kernel_file),
self._exists(self.system_fvm),
self._exists(self.zircona),
])
def _report_missing(self):
result = []
if not self._exists(self.build_args):
result.append(self.build_args)
if not self._exists(self.kernel_file):
result.append(self.kernel_file)
if not self._exists(self.system_fvm):
result.append(self.system_fvm)
if not self._exists(self.zircona):
result.append(self.zircona)
return result
@attr.s
class PackageFilePaths(object):
# Recipe API, required
_api = attr.ib(type=recipe_api.RecipeApi, init=True)
# Files from fuchsia packages
tar_file = attr.ib(type=Path, default=None)
amber_files = attr.ib(type=Path, default=None)
pm = attr.ib(type=Path, default=None)
def _exists(self, p):
return p and self._api.path.exists(p)
def _exist(self):
return all([
self._exists(self.tar_file),
self._exists(self.amber_files),
self._exists(self.pm),
])
def _report_missing(self):
result = []
if not self._exists(self.tar_file):
result.append(self.tar_file)
if not self._exists(self.amber_files):
result.append(self.amber_files)
if not self._exists(self.pm):
result.append(self.pm)
return result
class SDKApi(recipe_api.RecipeApi):
"""Downloads Fuchsia SDK files required to start FEMU."""
def __init__(self, *args, **kwargs):
super(SDKApi, self).__init__(*args, **kwargs)
self._image_paths = ImageFilePaths(api=self.m)
self._package_paths = PackageFilePaths(api=self.m)
self._sdk_path = None
self._version = None
def _fetch_sdk(self):
"""Downloads Fuchsia SDK from GCS and untar."""
with self.m.step.nest('ensure sdk'):
with self.m.context(infra_steps=True):
# Ensure cache path has the correct permission
cache_path = self.m.buildbucket.builder_cache_path.join(
self.version, 'fuchsia_sdk', self.platform_name
)
self.m.file.ensure_directory('init fuchsia_sdk cache', cache_path)
if not self.m.file.listdir(name='check sdk cache content',
source=cache_path, test_data=()):
# Copy GN image to temp directory
sdk_file = 'gn.tar.gz'
local_tmp_root = self.m.path.mkdtemp('fuchsia_sdk_tmp')
self.m.gsutil.download(
src_bucket=SDK_GCS_BUCKET,
src=self.m.path.join(
'development',
self.version,
'sdk',
self.sdk_platform_name,
sdk_file,
),
dest=local_tmp_root,
)
# Extract sdk
self.m.tar.extract(
step_name='extract sdk gz',
path=self.m.path.join(local_tmp_root, sdk_file),
directory=cache_path,
)
# Save cache path
self._sdk_path = cache_path
def _fetch_image(self):
"""Downloads Fuchsia image from GCS. Untar and store the required paths for FEMU."""
with self.m.step.nest('ensure image'):
with self.m.context(infra_steps=True):
image_to_download = self._select_image_to_download()
# Ensure cache path has the correct permission
cache_path = self.m.buildbucket.builder_cache_path.join(
self.version, 'fuchsia_image', self.platform_name
)
self.m.file.ensure_directory('init fuchsia_image cache', cache_path)
if not self.m.file.listdir(name='check image cache content',
source=cache_path, test_data=()):
# Copy image to temp directory
local_tmp_path = self.m.path.mkdtemp('fuchsia_image_tmp')
self.m.gsutil.download(
src_bucket=SDK_GCS_BUCKET,
src=self.m.path.join(
'development', self.version, 'images', image_to_download
),
dest=local_tmp_path,
)
# Extract image
self.m.tar.extract(
step_name='extract image tgz',
path=self.m.path.join(local_tmp_path, image_to_download),
directory=cache_path,
)
# Assemble files required for FEMU
for p in self.m.file.listdir(
name='set image files',
source=cache_path,
test_data=(
'buildargs.gn',
'qemu-kernel.kernel',
'zircon-a.zbi',
'zircon-r.zbi',
'storage-sparse.blk',
'storage-full.blk',
),
):
base = self.m.path.basename(p)
if base == 'buildargs.gn':
self._image_paths.build_args = p
elif base == 'qemu-kernel.kernel':
self._image_paths.kernel_file = p
elif base == 'storage-full.blk':
self._image_paths.system_fvm = p
elif base == 'zircon-a.zbi':
self._image_paths.zircona = p
def _fetch_packages(self):
with self.m.step.nest('ensure packages'):
with self.m.context(infra_steps=True):
package_to_download = self._select_package_to_download()
# Ensure cache path has the correct permission
cache_path = self.m.buildbucket.builder_cache_path.join(
self.version, 'fuchsia_packages', self.platform_name
)
self.m.file.ensure_directory('init fuchsia_packages cache', cache_path)
if not self.m.file.listdir(name='check packages cache content',
source=cache_path, test_data=()):
# Copy packages to cache directory.
# TODO(yuanzhi) Copy to tmp. We need to keep this in cache because fuchsia_ctl
# used by the flutter team expects packages as .tar.gz input. However, we should
# use fuchsia device controller (FDC) that comes with VDL for FEMU based testing
# eventually.
self.m.gsutil.download(
src_bucket=SDK_GCS_BUCKET,
src=self.m.path.join(
'development', self.version, 'packages', package_to_download
),
dest=cache_path,
)
# Extract package, this will produce the following subdirectories:
# |cache_path|
# |__ amber-files
# |__ keys
# |__ repository
# |__ pm
self.m.tar.extract(
step_name='extract package tar.gz',
path=self.m.path.join(cache_path, package_to_download),
directory=cache_path,
)
self._package_paths.tar_file = cache_path.join(package_to_download)
self._package_paths.amber_files = cache_path.join('amber-files')
self._package_paths.pm = cache_path.join('pm')
def authorize_zbi(
self,
ssh_key_path,
zbi_input_path,
zbi_output_path=None,
zbi_tool_path=None
):
"""Use zbi tool to extend BootFS with SSH authorization key.
Arguments:
ssh_key_path: path to public ssh key file.
zbi_input_path: path to zircon-a.zbi file.
zbi_output_path: (optional) output path to store extended zbi image file.
if None, we will replace zircon file specified in zbi_input_path
zbi_tool_path: (optional) path to the zbi binary tool.
if None, we will fetch the zbi tool from fuchsia sdk in GCS.
"""
zbi_path = None
if zbi_tool_path:
zbi_path = zbi_tool_path
if not zbi_path:
zbi_path = self.m.path.join(self.sdk_path, 'tools', 'zbi')
if not zbi_output_path:
zbi_output_path = zbi_input_path
self.m.step(
"authorize zbi",
[
zbi_path,
"--output",
self.m.raw_io.output_text(leak_to=zbi_output_path),
zbi_input_path,
"--entry",
"%s=%s" % ('data/ssh/authorized_keys', ssh_key_path),
],
)
def _select_package_to_download(self):
"""Maps platform parameters to Package names."""
return 'qemu-{arch}.tar.gz'.format(arch=self._select_arch())
def _select_image_to_download(self):
"""Maps platform parameters to Image names."""
return 'qemu-{arch}.tgz'.format(arch=self._select_arch())
def _select_arch(self):
"""Maps platform parameters to SDK names."""
if self.m.platform.arch == 'arm' and self.m.platform.bits == 64:
return 'arm64'
elif self.m.platform.arch == 'intel' and self.m.platform.bits == 64:
return 'x64'
raise self.m.step.StepFailure(
'Cannot find supported tools. arch %s, bit %s' %
(self.m.platform.arch, self.m.platform.bits)
)
@property
def sdk_platform_name(self):
"""Derives the sdk package platform name, resembles the CIPD platform name."""
name = ''
if self.m.platform.is_linux:
name = 'linux-amd64'
elif self.m.platform.is_mac:
name = 'mac-amd64'
return name
@property
def platform_name(self):
return '%s_%s_%s' % (
self.m.platform.name,
self.m.platform.arch,
self.m.platform.bits,
)
@property
def version(self):
return self._version
@version.setter
def version(self, value):
self._version = value
@property
def image_paths(self):
"""Downloads and unpacks Fuchsia image files from GCS.
Raises:
StepFailure: When cannot find image files matching host architecture.
StepFailure: When image files do not exist after download and unpack from GCS.
"""
assert self.version
self._fetch_image()
if not self._image_paths._exist():
missing = self._image_paths._report_missing()
ex = self.m.step.StepFailure(
'Image paths do not exist. {missing_paths}'.format(
missing_paths=missing
)
)
ex.missing_paths = missing
raise ex
return self._image_paths
@property
def package_paths(self):
"""Downloads and unpacks Fuchsia package files from GCS.
Raises:
StepFailure: When cannot find package files matching host architecture.
StepFailure: When package files do not exist after download and unpack from GCS.
"""
assert self.version
self._fetch_packages()
if not self._package_paths._exist():
missing = self._package_paths._report_missing()
ex = self.m.step.StepFailure(
'Package paths do not exist. {missing_paths}'.format(
missing_paths=missing
)
)
ex.missing_paths = missing
raise ex
return self._package_paths
@property
def sdk_path(self):
"""Downloads and unpacks Fuchsia sdk files from GCS.
Raises:
StepFailure: When cannot find sdk files matching host architecture.
"""
assert self.version
self._fetch_sdk()
return self._sdk_path