create module for kms secretes This cl updates to use infra kms module directly, preparing for luci tasks with secretes. Change-Id: I5eb648fdfe163c86c6546a627bbf084566155fb5 Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/3760 Reviewed-by: Godofredo Contreras <godofredoc@google.com> Commit-Queue: Keyong Han <keyonghan@google.com>
diff --git a/recipe_modules/kms/__init__.py b/recipe_modules/kms/__init__.py new file mode 100644 index 0000000..af64233 --- /dev/null +++ b/recipe_modules/kms/__init__.py
@@ -0,0 +1,8 @@ +DEPS = [ + 'depot_tools/gsutil', + 'recipe_engine/cipd', + 'recipe_engine/context', + 'recipe_engine/path', + 'recipe_engine/platform', + 'recipe_engine/step', +]
diff --git a/recipe_modules/kms/api.py b/recipe_modules/kms/api.py new file mode 100644 index 0000000..638bc81 --- /dev/null +++ b/recipe_modules/kms/api.py
@@ -0,0 +1,28 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine import recipe_api + +class KMSApi(recipe_api.RecipeApi): + """Provides KMS support for recipe secretes.""" + def get_secret(self, input_file, secret_path): + """Decrypts the encrypted secret. + + Args: + input_file (str): encrypted file of the secret. + secret_path (Path): path of decrypted secret. + """ + cloudkms_dir = self.m.path['start_dir'].join('cloudkms') + cloudkms_package = 'infra/tools/luci/cloudkms/${platform}' + self.m.cipd.ensure( + cloudkms_dir, + self.m.cipd.EnsureFile().add_package(cloudkms_package, 'latest')) + encrypt_file = self.m.path['cleanup'].join(input_file) + self.m.gsutil.download('flutter_configs', input_file, encrypt_file) + cloudkms = cloudkms_dir.join('cloudkms.exe' if self.m.platform.name == 'win' else 'cloudkms') + self.m.step('cloudkms get key', [ + cloudkms, 'decrypt', '-input', encrypt_file, '-output', secret_path, + 'projects/flutter-infra/locations/global' + '/keyRings/luci/cryptoKeys/flutter-infra' + ]) \ No newline at end of file
diff --git a/recipe_modules/kms/examples/get_secret.expected/basic.json b/recipe_modules/kms/examples/get_secret.expected/basic.json new file mode 100644 index 0000000..481772e --- /dev/null +++ b/recipe_modules/kms/examples/get_secret.expected/basic.json
@@ -0,0 +1,60 @@ +[ + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[START_DIR]/cloudkms", + "-ensure-file", + "infra/tools/luci/cloudkms/${platform} latest", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cloudkms/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_configs/in", + "[CLEANUP]/in" + ], + "infra_step": true, + "name": "gsutil download" + }, + { + "cmd": [ + "[START_DIR]/cloudkms/cloudkms", + "decrypt", + "-input", + "[CLEANUP]/in", + "-output", + "[CLEANUP]/out", + "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/flutter-infra" + ], + "name": "cloudkms get key" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/kms/examples/get_secret.expected/win.json b/recipe_modules/kms/examples/get_secret.expected/win.json new file mode 100644 index 0000000..89e34a6 --- /dev/null +++ b/recipe_modules/kms/examples/get_secret.expected/win.json
@@ -0,0 +1,14 @@ +[ + { + "cmd": [ + "[START_DIR]\\cloudkms\\cloudkms.exe", + "decrypt", + "-input", + "[CLEANUP]\\in", + "-output", + "[CLEANUP]\\out", + "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/flutter-infra" + ], + "name": "cloudkms get key" + } +] \ No newline at end of file
diff --git a/recipe_modules/kms/examples/get_secret.py b/recipe_modules/kms/examples/get_secret.py new file mode 100644 index 0000000..a1c2f6c --- /dev/null +++ b/recipe_modules/kms/examples/get_secret.py
@@ -0,0 +1,22 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from recipe_engine.post_process import (Filter) + +DEPS = [ + 'kms', + 'recipe_engine/path', + 'recipe_engine/platform', +] + +def RunSteps(api): + api.kms.get_secret('in', api.path['cleanup'].join('out')) + +def GenTests(api): + yield api.test('basic') + yield api.test( + 'win', + api.platform('win', 64), + api.post_process(Filter('cloudkms get key')), + ) \ No newline at end of file
diff --git a/recipes/flutter.expected/linux_master_coverage_.json b/recipes/flutter.expected/linux_master_coverage_.json index fab7d4b..dc1047c 100644 --- a/recipes/flutter.expected/linux_master_coverage_.json +++ b/recipes/flutter.expected/linux_master_coverage_.json
@@ -197,7 +197,7 @@ "GOLDCTL": "[CACHE]/gold/goldctl", "JAVA_HOME": "[CACHE]/java", "PUB_CACHE": "[START_DIR]/flutter/.pub-cache", - "SHARD": "coverage" + "SHARD": "framework_coverage" }, "env_prefixes": { "PATH": [ @@ -206,7 +206,7 @@ "[CACHE]/java/bin" ] }, - "name": "run test.dart for coverage shard" + "name": "run test.dart for framework_coverage shard" }, { "name": "$result"
diff --git a/recipes/flutter.expected/linux_master_coverage__experimental.json b/recipes/flutter.expected/linux_master_coverage__experimental.json index fab7d4b..dc1047c 100644 --- a/recipes/flutter.expected/linux_master_coverage__experimental.json +++ b/recipes/flutter.expected/linux_master_coverage__experimental.json
@@ -197,7 +197,7 @@ "GOLDCTL": "[CACHE]/gold/goldctl", "JAVA_HOME": "[CACHE]/java", "PUB_CACHE": "[START_DIR]/flutter/.pub-cache", - "SHARD": "coverage" + "SHARD": "framework_coverage" }, "env_prefixes": { "PATH": [ @@ -206,7 +206,7 @@ "[CACHE]/java/bin" ] }, - "name": "run test.dart for coverage shard" + "name": "run test.dart for framework_coverage shard" }, { "name": "$result"
diff --git a/recipes/flutter.expected/linux_master_coverage__experimental_upload.json b/recipes/flutter.expected/linux_master_coverage__experimental_upload.json index 3815aa4..0899b2f 100644 --- a/recipes/flutter.expected/linux_master_coverage__experimental_upload.json +++ b/recipes/flutter.expected/linux_master_coverage__experimental_upload.json
@@ -197,7 +197,7 @@ "GOLDCTL": "[CACHE]/gold/goldctl", "JAVA_HOME": "[CACHE]/java", "PUB_CACHE": "[START_DIR]/flutter/.pub-cache", - "SHARD": "coverage" + "SHARD": "framework_coverage" }, "env_prefixes": { "PATH": [ @@ -206,7 +206,7 @@ "[CACHE]/java/bin" ] }, - "name": "run test.dart for coverage shard" + "name": "run test.dart for framework_coverage shard" }, { "cmd": [ @@ -241,15 +241,11 @@ ] }, { - "cmd": [], - "name": "ensure_cloudkms" - }, - { "cmd": [ "cipd", "ensure", "-root", - "[START_DIR]/cipd/cloudkms", + "[START_DIR]/cloudkms", "-ensure-file", "infra/tools/luci/cloudkms/${platform} latest", "-max-threads", @@ -271,10 +267,8 @@ "[CACHE]/java/bin" ] }, - "infra_step": true, - "name": "ensure_cloudkms.ensure_installed", + "name": "ensure_installed (2)", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@json.output@{@@@", "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", @@ -290,13 +284,15 @@ }, { "cmd": [ - "[START_DIR]/cipd/cloudkms/cloudkms", - "decrypt", - "-input", - "RECIPE[flutter::flutter].resources/coveralls-token.enc", - "-output", - "[START_DIR]/flutter/packages/flutter/.coveralls.yml", - "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/coveralls" + "python", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_configs/codecov.encrypted", + "[CLEANUP]/codecov.encrypted" ], "cwd": "[START_DIR]/flutter", "env": { @@ -312,7 +308,34 @@ "[CACHE]/java/bin" ] }, - "name": "decrypt coveralls token" + "infra_step": true, + "name": "gsutil download" + }, + { + "cmd": [ + "[START_DIR]/cloudkms/cloudkms", + "decrypt", + "-input", + "[CLEANUP]/codecov.encrypted", + "-output", + "[START_DIR]/flutter/packages/flutter/.coveralls.yml", + "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/flutter-infra" + ], + "cwd": "[START_DIR]/flutter", + "env": { + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "GOLDCTL": "[CACHE]/gold/goldctl", + "JAVA_HOME": "[CACHE]/java", + "PUB_CACHE": "[START_DIR]/flutter/.pub-cache" + }, + "env_prefixes": { + "PATH": [ + "[START_DIR]/flutter/bin", + "[START_DIR]/flutter/bin/cache/dart-sdk/bin", + "[CACHE]/java/bin" + ] + }, + "name": "cloudkms get key" }, { "cmd": [
diff --git a/recipes/flutter.expected/linux_master_coverage__upload.json b/recipes/flutter.expected/linux_master_coverage__upload.json index bca8cb3..4a4baa3 100644 --- a/recipes/flutter.expected/linux_master_coverage__upload.json +++ b/recipes/flutter.expected/linux_master_coverage__upload.json
@@ -197,7 +197,7 @@ "GOLDCTL": "[CACHE]/gold/goldctl", "JAVA_HOME": "[CACHE]/java", "PUB_CACHE": "[START_DIR]/flutter/.pub-cache", - "SHARD": "coverage" + "SHARD": "framework_coverage" }, "env_prefixes": { "PATH": [ @@ -206,7 +206,7 @@ "[CACHE]/java/bin" ] }, - "name": "run test.dart for coverage shard" + "name": "run test.dart for framework_coverage shard" }, { "cmd": [ @@ -241,15 +241,11 @@ ] }, { - "cmd": [], - "name": "ensure_cloudkms" - }, - { "cmd": [ "cipd", "ensure", "-root", - "[START_DIR]/cipd/cloudkms", + "[START_DIR]/cloudkms", "-ensure-file", "infra/tools/luci/cloudkms/${platform} latest", "-max-threads", @@ -271,10 +267,8 @@ "[CACHE]/java/bin" ] }, - "infra_step": true, - "name": "ensure_cloudkms.ensure_installed", + "name": "ensure_installed (2)", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", "@@@STEP_LOG_LINE@json.output@{@@@", "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", @@ -290,13 +284,15 @@ }, { "cmd": [ - "[START_DIR]/cipd/cloudkms/cloudkms", - "decrypt", - "-input", - "RECIPE[flutter::flutter].resources/coveralls-token.enc", - "-output", - "[START_DIR]/flutter/packages/flutter/.coveralls.yml", - "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/coveralls" + "python", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_configs/codecov.encrypted", + "[CLEANUP]/codecov.encrypted" ], "cwd": "[START_DIR]/flutter", "env": { @@ -312,7 +308,34 @@ "[CACHE]/java/bin" ] }, - "name": "decrypt coveralls token" + "infra_step": true, + "name": "gsutil download" + }, + { + "cmd": [ + "[START_DIR]/cloudkms/cloudkms", + "decrypt", + "-input", + "[CLEANUP]/codecov.encrypted", + "-output", + "[START_DIR]/flutter/packages/flutter/.coveralls.yml", + "projects/flutter-infra/locations/global/keyRings/luci/cryptoKeys/flutter-infra" + ], + "cwd": "[START_DIR]/flutter", + "env": { + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "GOLDCTL": "[CACHE]/gold/goldctl", + "JAVA_HOME": "[CACHE]/java", + "PUB_CACHE": "[START_DIR]/flutter/.pub-cache" + }, + "env_prefixes": { + "PATH": [ + "[START_DIR]/flutter/bin", + "[START_DIR]/flutter/bin/cache/dart-sdk/bin", + "[CACHE]/java/bin" + ] + }, + "name": "cloudkms get key" }, { "cmd": [
diff --git a/recipes/flutter.py b/recipes/flutter.py index 2afbd0a..0f4f7c3 100644 --- a/recipes/flutter.py +++ b/recipes/flutter.py
@@ -11,6 +11,7 @@ 'depot_tools/gsutil', 'depot_tools/osx_sdk', 'depot_tools/windows_sdk', + 'flutter/kms', 'flutter/zip', 'fuchsia/display_util', 'recipe_engine/buildbucket', @@ -91,32 +92,6 @@ env={'JAVA_HOME': java_cache_dir}, env_prefixes={'PATH': [java_cache_dir.join('bin')]}) - -def EnsureCloudKMS(api, version=None): - with api.step.nest('ensure_cloudkms'): - with api.context(infra_steps=True): - pkgs = api.cipd.EnsureFile() - pkgs.add_package('infra/tools/luci/cloudkms/${platform}', version or - 'latest') - cipd_dir = api.path['start_dir'].join('cipd', 'cloudkms') - api.cipd.ensure(cipd_dir, pkgs) - return cipd_dir.join('cloudkms') - - -def DecryptKMS(api, step_name, crypto_key_path, ciphertext_file, - plaintext_file): - kms_path = EnsureCloudKMS(api) - return api.step(step_name, [ - kms_path, - 'decrypt', - '-input', - ciphertext_file, - '-output', - plaintext_file, - crypto_key_path, - ]) - - def GetCloudPath(api, git_hash, path): if api.runtime.is_experimental: return 'flutter/experimental/%s/%s' % (git_hash, path) @@ -140,12 +115,10 @@ link_name='lcov.info', name='upload coverage data') + # Download encrpted file to local and then decrypt. token_path = flutter_package_dir.join('.coveralls.yml') - DecryptKMS(api, 'decrypt coveralls token', - 'projects/flutter-infra/locations/global' \ - '/keyRings/luci/cryptoKeys/coveralls', - api.resource('coveralls-token.enc'), - token_path) + api.kms.get_secret('codecov.encrypted', token_path) + pub_executable = 'pub' if not api.platform.is_win else 'pub.exe' api.step('pub global activate coveralls', [ pub_executable, 'global', 'activate', 'coveralls', '5.1.0', @@ -260,7 +233,7 @@ api.step('run test.dart for %s shard' % shard, [dart_executable, checkout.join('dev', 'bots', 'test.dart')]) - if shard == 'coverage': + if shard == 'framework_coverage': UploadFlutterCoverage(api) # Windows uses exclusive file locking. On LUCI, if these processes remain # they will cause the build to fail because the builder won't be able to @@ -288,7 +261,7 @@ '_upload' if should_upload else ''), api.runtime(is_luci=True, is_experimental=experimental), api.properties( - shard='coverage', + shard='framework_coverage', coveralls_lcov_version='5.1.0', upload_packages=should_upload, gold_tryjob=not should_upload),