Preprocess json properties before passing to subbuilds.
Ci yaml encodes dictionary and list properties as json_strings and we
need to convert them back to list or dicts before passing to subbuilds.
Change-Id: Ided452ea0a7e0f42f92a6b7c14d4b66f3e863e7d
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/36340
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
Reviewed-by: Ricardo Amador <ricardoamador@google.com>
diff --git a/recipe_modules/shard_util_v2/api.py b/recipe_modules/shard_util_v2/api.py
index 61f566e..bdfca30 100644
--- a/recipe_modules/shard_util_v2/api.py
+++ b/recipe_modules/shard_util_v2/api.py
@@ -3,6 +3,7 @@
# found in the LICENSE file.
import attr
+import json
import collections
from google.protobuf import duration_pb2
@@ -72,6 +73,24 @@
result[k] = v
return result
+ def pre_process_properties(self, target):
+ """Converts json properties to dicts or lists.
+
+ Dict or lists in ci_yaml are passed as json string to recipes and they
+ need to be converted back to dict or lists before passing them to subbuilds.
+
+ Args:
+ target: A target dictionary as read from the yaml file.
+ """
+ if target.get('properties'):
+ properties = target.get('properties')
+ new_props = {}
+ for k, v in properties.items():
+ if isinstance(v,str) and (v.startswith('[') or v.startswith('{')):
+ new_props[k] = json.loads(v)
+ target['properties'] = new_props
+ return target
+
def struct_to_dict(self, struct):
"""Transforms a proto structure to a dictionary.
diff --git a/recipe_modules/shard_util_v2/examples/full.py b/recipe_modules/shard_util_v2/examples/full.py
index d1cf361..bbe4a92 100644
--- a/recipe_modules/shard_util_v2/examples/full.py
+++ b/recipe_modules/shard_util_v2/examples/full.py
@@ -20,6 +20,13 @@
def RunSteps(api):
build_configs = api.properties.get('builds', [])
test_configs = api.properties.get('tests', [])
+ props = api.shard_util_v2.pre_process_properties(
+ {'properties': {
+ '$flutter/osx_sdk': '{"cleanup_cache": true, "sdk_version": "14a5294e"}'
+ }
+ }
+ )
+ assert isinstance(props['properties']['$flutter/osx_sdk'], dict)
with api.step.nest("launch builds") as presentation:
reqs = api.shard_util_v2.schedule_builds(build_configs, presentation)
with api.step.nest("collect builds") as presentation:
diff --git a/recipes/engine/release_builder.expected/basic_linux.json b/recipes/engine/release_builder.expected/basic_linux.json
index d2d7b86..fdc3ab4 100644
--- a/recipes/engine/release_builder.expected/basic_linux.json
+++ b/recipes/engine/release_builder.expected/basic_linux.json
@@ -261,6 +261,7 @@
"@@@STEP_LOG_LINE@json.output@ {@@@",
"@@@STEP_LOG_LINE@json.output@ \"name\": \"linux one\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"$flutter/osx_sdk\": \"{\\\"sdk_version\\\": \\\"14a5294e\\\"}\", @@@",
"@@@STEP_LOG_LINE@json.output@ \"release_build\": true@@@",
"@@@STEP_LOG_LINE@json.output@ }, @@@",
"@@@STEP_LOG_LINE@json.output@ \"recipe\": \"engine/something\"@@@",
@@ -295,7 +296,7 @@
}
},
"name": "launch builds.schedule",
- "stdin": "{\"requests\": [{\"scheduleBuild\": {\"builder\": {\"bucket\": \"try\", \"builder\": \"Linux Engine Drone\", \"project\": \"proj\"}, \"exe\": {\"cipdVersion\": \"refs/heads/main\"}, \"experimental\": \"NO\", \"experiments\": {\"luci.buildbucket.parent_tracking\": false}, \"fields\": \"builder,createTime,createdBy,critical,endTime,id,infra,input,number,output,startTime,status,updateTime\", \"gerritChanges\": [{\"change\": \"123456\", \"host\": \"flutter-review.googlesource.com\", \"patchset\": \"7\", \"project\": \"mirrors/engine\"}], \"gitilesCommit\": {\"host\": \"flutter.googlesource.com\", \"id\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"project\": \"mirrors/engine\", \"ref\": \"refs/heads/main\"}, \"priority\": 25, \"properties\": {\"build\": {\"name\": \"linux one\", \"properties\": {\"release_build\": true}, \"recipe\": \"engine/something\"}, \"environment\": \"Staging\", \"recipe\": \"engine/something\", \"release_build\": true}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"cq_experimental\", \"value\": \"false\"}, {\"key\": \"parent_buildbucket_id\", \"value\": \"8945511751514863184\"}, {\"key\": \"user_agent\", \"value\": \"recipe\"}]}}]}",
+ "stdin": "{\"requests\": [{\"scheduleBuild\": {\"builder\": {\"bucket\": \"try\", \"builder\": \"Linux Engine Drone\", \"project\": \"proj\"}, \"exe\": {\"cipdVersion\": \"refs/heads/main\"}, \"experimental\": \"NO\", \"experiments\": {\"luci.buildbucket.parent_tracking\": false}, \"fields\": \"builder,createTime,createdBy,critical,endTime,id,infra,input,number,output,startTime,status,updateTime\", \"gerritChanges\": [{\"change\": \"123456\", \"host\": \"flutter-review.googlesource.com\", \"patchset\": \"7\", \"project\": \"mirrors/engine\"}], \"gitilesCommit\": {\"host\": \"flutter.googlesource.com\", \"id\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"project\": \"mirrors/engine\", \"ref\": \"refs/heads/main\"}, \"priority\": 25, \"properties\": {\"$flutter/osx_sdk\": {\"sdk_version\": \"14a5294e\"}, \"build\": {\"name\": \"linux one\", \"properties\": {\"$flutter/osx_sdk\": {\"sdk_version\": \"14a5294e\"}}, \"recipe\": \"engine/something\"}, \"environment\": \"Staging\", \"recipe\": \"engine/something\"}, \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", \"swarming\": {\"parentRunId\": \"fake-task-id\"}, \"tags\": [{\"key\": \"cq_experimental\", \"value\": \"false\"}, {\"key\": \"parent_buildbucket_id\", \"value\": \"8945511751514863184\"}, {\"key\": \"user_agent\", \"value\": \"recipe\"}]}}]}",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
@@ -346,16 +347,20 @@
"@@@STEP_LOG_LINE@request@ }, @@@",
"@@@STEP_LOG_LINE@request@ \"priority\": 25, @@@",
"@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"$flutter/osx_sdk\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_version\": \"14a5294e\"@@@",
+ "@@@STEP_LOG_LINE@request@ }, @@@",
"@@@STEP_LOG_LINE@request@ \"build\": {@@@",
"@@@STEP_LOG_LINE@request@ \"name\": \"linux one\", @@@",
"@@@STEP_LOG_LINE@request@ \"properties\": {@@@",
- "@@@STEP_LOG_LINE@request@ \"release_build\": true@@@",
+ "@@@STEP_LOG_LINE@request@ \"$flutter/osx_sdk\": {@@@",
+ "@@@STEP_LOG_LINE@request@ \"sdk_version\": \"14a5294e\"@@@",
+ "@@@STEP_LOG_LINE@request@ }@@@",
"@@@STEP_LOG_LINE@request@ }, @@@",
"@@@STEP_LOG_LINE@request@ \"recipe\": \"engine/something\"@@@",
"@@@STEP_LOG_LINE@request@ }, @@@",
"@@@STEP_LOG_LINE@request@ \"environment\": \"Staging\", @@@",
- "@@@STEP_LOG_LINE@request@ \"recipe\": \"engine/something\", @@@",
- "@@@STEP_LOG_LINE@request@ \"release_build\": true@@@",
+ "@@@STEP_LOG_LINE@request@ \"recipe\": \"engine/something\"@@@",
"@@@STEP_LOG_LINE@request@ }, @@@",
"@@@STEP_LOG_LINE@request@ \"requestId\": \"8945511751514863184-00000000-0000-0000-0000-000000001337\", @@@",
"@@@STEP_LOG_LINE@request@ \"swarming\": {@@@",
diff --git a/recipes/engine/release_builder.py b/recipes/engine/release_builder.py
index bb5ab00..ec8866b 100644
--- a/recipes/engine/release_builder.py
+++ b/recipes/engine/release_builder.py
@@ -8,6 +8,7 @@
# marked with release_build: true, and spawens a subbuild.
+import json
from contextlib import contextmanager
from PB.recipes.flutter.engine.engine import InputProperties
@@ -57,6 +58,7 @@
for target in ci_yaml.json.output['targets']:
if target.get("properties", {}).get("release_build", False) and (
target["name"].lower().startswith(api.platform.name)):
+ target = api.shard_util_v2.pre_process_properties(target)
tasks.update(api.shard_util_v2.schedule(
[target, ], target["recipe"], presentation))
with api.step.nest('collect builds') as presentation:
@@ -77,7 +79,10 @@
status="SUCCESS",
)
tasks_dict = {'targets': [
- {'name': 'linux one', 'recipe': 'engine/something', 'properties': {'release_build': True, }}]}
+ {'name': 'linux one', 'recipe': 'engine/something',
+ 'properties': {'release_build': True, '$flutter/osx_sdk': '{"sdk_version": "14a5294e"}'}
+ }]
+ }
yield api.test(
'basic_linux',
api.platform.name('linux'),
