Lalit Maganti | df50009 | 2020-07-16 23:56:24 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 2 | # Copyright (C) 2018 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
Florian Mayer | cdc44b3 | 2020-06-29 17:18:29 +0100 | [diff] [blame] | 16 | from __future__ import absolute_import |
| 17 | from __future__ import division |
| 18 | from __future__ import print_function |
| 19 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 20 | import argparse |
Lalit Maganti | 467a8ef | 2019-12-05 15:54:20 +0000 | [diff] [blame] | 21 | import datetime |
Lalit Maganti | 46a0a53 | 2019-07-15 15:09:22 +0100 | [diff] [blame] | 22 | import json |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 23 | import os |
Lalit Maganti | 4be82e1 | 2019-12-05 15:33:09 +0000 | [diff] [blame] | 24 | import re |
Primiano Tucci | 521d620 | 2022-07-29 11:41:25 +0100 | [diff] [blame] | 25 | import signal |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 26 | import sys |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 27 | |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 28 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
Anna Mayzner | 6e723ce | 2023-01-09 17:03:13 +0000 | [diff] [blame] | 29 | sys.path.append(os.path.join(ROOT_DIR)) |
| 30 | |
Anna Mayzner | a0d461d | 2023-01-13 16:45:45 +0000 | [diff] [blame] | 31 | from python.generators.diff_tests.testing import TestType |
Anna Mayzner | f60f53d | 2023-01-13 15:46:00 +0000 | [diff] [blame] | 32 | from python.generators.diff_tests.utils import ctrl_c_handler |
Anna Mayzner | 6988df8 | 2023-01-23 10:37:16 +0000 | [diff] [blame] | 33 | from python.generators.diff_tests.runner import DiffTestsRunner |
Primiano Tucci | 521d620 | 2022-07-29 11:41:25 +0100 | [diff] [blame] | 34 | |
| 35 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 36 | def main(): |
Primiano Tucci | 521d620 | 2022-07-29 11:41:25 +0100 | [diff] [blame] | 37 | signal.signal(signal.SIGINT, ctrl_c_handler) |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 38 | parser = argparse.ArgumentParser() |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 39 | parser.add_argument('--test-type', type=str, default='all') |
Lalit Maganti | da1c1d0 | 2019-05-08 11:55:50 +0100 | [diff] [blame] | 40 | parser.add_argument('--trace-descriptor', type=str) |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 41 | parser.add_argument('--metrics-descriptor', nargs='+', type=str) |
| 42 | parser.add_argument('--chrome-track-event-descriptor', type=str, default=None) |
| 43 | parser.add_argument('--test-extensions', type=str, default=None) |
Kean Mariotti | b4a6d11 | 2024-04-11 13:57:22 +0000 | [diff] [blame] | 44 | parser.add_argument('--winscope-extensions', type=str, default=None) |
Lalit Maganti | 75869e6 | 2019-07-12 16:55:38 +0100 | [diff] [blame] | 45 | parser.add_argument('--perf-file', type=str) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 46 | parser.add_argument( |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 47 | '--override-sql-module', type=str, action='append', default=[]) |
| 48 | parser.add_argument('--test-dir', type=str, default=ROOT_DIR) |
| 49 | parser.add_argument( |
Anna Mayzner | 73148f7 | 2023-01-24 16:32:16 +0000 | [diff] [blame] | 50 | '--name-filter', |
Lalit Maganti | 4be82e1 | 2019-12-05 15:33:09 +0000 | [diff] [blame] | 51 | default='.*', |
| 52 | type=str, |
Anna Mayzner | 73148f7 | 2023-01-24 16:32:16 +0000 | [diff] [blame] | 53 | help='Filter the name of the tests to run (regex syntax)') |
Lalit Maganti | 4be82e1 | 2019-12-05 15:33:09 +0000 | [diff] [blame] | 54 | parser.add_argument( |
Eric Seckler | fbd9aed | 2020-03-10 18:07:38 +0000 | [diff] [blame] | 55 | '--keep-input', |
| 56 | action='store_true', |
| 57 | help='Save the (generated) input pb file for debugging') |
| 58 | parser.add_argument( |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 59 | '--rebase', |
| 60 | action='store_true', |
| 61 | help='Update the expected output file with the actual result') |
| 62 | parser.add_argument( |
Anna Mayzner | ceaa8b8 | 2024-11-06 13:35:23 +0000 | [diff] [blame] | 63 | '--quiet', action='store_true', help='Only print if the test failed.') |
| 64 | parser.add_argument( |
Anna Mayzner | 364c0c9 | 2022-07-08 17:51:05 +0000 | [diff] [blame] | 65 | '--no-colors', action='store_true', help='Print without coloring') |
| 66 | parser.add_argument( |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 67 | 'trace_processor', type=str, help='location of trace processor binary') |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 68 | args = parser.parse_args() |
| 69 | |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 70 | out_path = os.path.dirname(args.trace_processor) |
| 71 | if args.chrome_track_event_descriptor is None: |
| 72 | args.chrome_track_event_descriptor = os.path.join( |
| 73 | out_path, 'gen', 'protos', 'third_party', 'chromium', |
| 74 | 'chrome_track_event.descriptor') |
| 75 | if args.test_extensions is None: |
| 76 | args.test_extensions = os.path.join(out_path, 'gen', 'protos', 'perfetto', |
| 77 | 'trace', 'test_extensions.descriptor') |
Kean Mariotti | b4a6d11 | 2024-04-11 13:57:22 +0000 | [diff] [blame] | 78 | if args.winscope_extensions is None: |
Lalit Maganti | bdd33a2 | 2024-05-29 17:58:14 +0100 | [diff] [blame] | 79 | args.winscope_extensions = os.path.join(out_path, 'gen', 'protos', |
| 80 | 'perfetto', 'trace', 'android', |
| 81 | 'winscope.descriptor') |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 82 | |
Anna Mayzner | 73148f7 | 2023-01-24 16:32:16 +0000 | [diff] [blame] | 83 | test_runner = DiffTestsRunner(args.name_filter, args.trace_processor, |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 84 | args.trace_descriptor, args.no_colors, |
Anna Mayzner | ceaa8b8 | 2024-11-06 13:35:23 +0000 | [diff] [blame] | 85 | args.override_sql_module, args.test_dir, |
| 86 | args.quiet) |
Anna Mayzner | f60f53d | 2023-01-13 15:46:00 +0000 | [diff] [blame] | 87 | sys.stderr.write(f"[==========] Running {len(test_runner.tests)} tests.\n") |
Lalit Maganti | 0ec47c6 | 2020-02-12 16:24:08 +0000 | [diff] [blame] | 88 | |
Rasika Navarange | 6398113 | 2023-07-05 11:26:50 +0000 | [diff] [blame] | 89 | results = test_runner.run_all_tests(args.metrics_descriptor, |
Lalit Maganti | bdd33a2 | 2024-05-29 17:58:14 +0100 | [diff] [blame] | 90 | args.chrome_track_event_descriptor, |
| 91 | args.test_extensions, |
| 92 | args.winscope_extensions, args.keep_input, |
| 93 | args.rebase) |
Anna Mayzner | f60f53d | 2023-01-13 15:46:00 +0000 | [diff] [blame] | 94 | sys.stderr.write(results.str(args.no_colors, len(test_runner.tests))) |
Anna Mayzner | 364c0c9 | 2022-07-08 17:51:05 +0000 | [diff] [blame] | 95 | |
Dan Elphick | e603bf7 | 2020-12-21 16:01:06 +0000 | [diff] [blame] | 96 | if args.rebase: |
Anna Mayzner | f60f53d | 2023-01-13 15:46:00 +0000 | [diff] [blame] | 97 | sys.stderr.write(results.rebase_str()) |
Lalit Maganti | 04a450a | 2019-07-03 16:04:19 +0100 | [diff] [blame] | 98 | |
Anna Mayzner | f60f53d | 2023-01-13 15:46:00 +0000 | [diff] [blame] | 99 | if len(results.test_failures) > 0: |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 100 | return 1 |
| 101 | |
Ryan Savitski | a6aecee | 2022-10-03 12:36:30 +0100 | [diff] [blame] | 102 | if args.perf_file: |
| 103 | test_dir = os.path.join(ROOT_DIR, 'test') |
| 104 | trace_processor_dir = os.path.join(test_dir, 'trace_processor') |
| 105 | |
| 106 | metrics = [] |
| 107 | sorted_data = sorted( |
Anna Mayzner | 95b5b75 | 2023-01-19 13:32:44 +0000 | [diff] [blame] | 108 | results.perf_data, key=lambda x: (x.test.type.name, x.test.name)) |
Ryan Savitski | a6aecee | 2022-10-03 12:36:30 +0100 | [diff] [blame] | 109 | for perf_args in sorted_data: |
Ryan Savitski | a6aecee | 2022-10-03 12:36:30 +0100 | [diff] [blame] | 110 | metrics.append({ |
| 111 | 'metric': 'tp_perf_test_ingest_time', |
| 112 | 'value': float(perf_args.ingest_time_ns) / 1.0e9, |
| 113 | 'unit': 's', |
| 114 | 'tags': { |
Anna Mayzner | 95b5b75 | 2023-01-19 13:32:44 +0000 | [diff] [blame] | 115 | 'test_name': perf_args.test.name, |
| 116 | 'test_type': perf_args.test.type.name, |
Ryan Savitski | a6aecee | 2022-10-03 12:36:30 +0100 | [diff] [blame] | 117 | }, |
| 118 | 'labels': {}, |
| 119 | }) |
| 120 | metrics.append({ |
| 121 | 'metric': 'perf_test_real_time', |
| 122 | 'value': float(perf_args.real_time_ns) / 1.0e9, |
| 123 | 'unit': 's', |
| 124 | 'tags': { |
Anna Mayzner | 95b5b75 | 2023-01-19 13:32:44 +0000 | [diff] [blame] | 125 | 'test_name': perf_args.test.name, |
| 126 | 'test_type': perf_args.test.type.name, |
Ryan Savitski | a6aecee | 2022-10-03 12:36:30 +0100 | [diff] [blame] | 127 | }, |
| 128 | 'labels': {}, |
| 129 | }) |
| 130 | |
| 131 | output_data = {'metrics': metrics} |
| 132 | with open(args.perf_file, 'w+') as perf_file: |
| 133 | perf_file.write(json.dumps(output_data, indent=2)) |
| 134 | return 0 |
| 135 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 136 | |
Lalit Maganti | 21160e8 | 2018-10-16 09:40:29 +0100 | [diff] [blame] | 137 | if __name__ == '__main__': |
| 138 | sys.exit(main()) |