Diff report for screenshot tests
Change-Id: I0037f901762348b1e9ac5447639563fba199353b
diff --git a/infra/ci/controller/controller.py b/infra/ci/controller/controller.py
index 7647fed..d5d2afe 100644
--- a/infra/ci/controller/controller.py
+++ b/infra/ci/controller/controller.py
@@ -318,6 +318,9 @@
if '-ui-' in job_id:
ui_links.append('https://storage.googleapis.com/%s/%s/ui/index.html' %
(GCS_ARTIFACTS, job_id))
+ ui_links.append(
+ 'https://storage.googleapis.com/%s/%s/ui-test-artifacts/index.html' %
+ (GCS_ARTIFACTS, job_id))
if job_obj['status'] == 'COMPLETED':
passed_jobs.append(job_id)
elif not job_config.get('SKIP_VOTING', False):
diff --git a/ui/build.js b/ui/build.js
index 758dee6..8a2ad50 100644
--- a/ui/build.js
+++ b/ui/build.js
@@ -100,6 +100,7 @@
outDir: pjoin(ROOT_DIR, 'out/ui'),
version: '', // v1.2.3, derived from the CHANGELOG + git.
outUiDir: '',
+ outUiTestArtifactsDir: '',
outDistRootDir: '',
outTscDir: '',
outGenDir: '',
@@ -115,6 +116,10 @@
{r: /ui\/src\/assets\/.+[.]scss/, f: compileScss},
{r: /ui\/src\/assets\/.+[.]scss/, f: compileScss},
{r: /ui\/src\/chrome_extension\/.*/, f: copyExtensionAssets},
+ {
+ r: /ui\/src\/test\/diff_viewer\/(.+[.](?:html|js))/,
+ f: copyUiTestArtifactsAssets,
+ },
{r: /.*\/dist\/.+\/(?!manifest\.json).*/, f: genServiceWorkerManifestJson},
{r: /.*\/dist\/.*/, f: notifyLiveServer},
];
@@ -150,6 +155,7 @@
const clean = !args.no_build;
cfg.outDir = path.resolve(ensureDir(args.out || cfg.outDir));
cfg.outUiDir = ensureDir(pjoin(cfg.outDir, 'ui'), clean);
+ cfg.outUiTestArtifactsDir = ensureDir(pjoin(cfg.outDir, 'ui-test-artifacts'));
cfg.outExtDir = ensureDir(pjoin(cfg.outUiDir, 'chrome_extension'));
cfg.outDistRootDir = ensureDir(pjoin(cfg.outUiDir, 'dist'));
const proc = exec('python3', [VERSION_SCRIPT, '--stdout'], {stdout: 'pipe'});
@@ -219,6 +225,7 @@
buildWasm(args.no_wasm);
scanDir('ui/src/assets');
scanDir('ui/src/chrome_extension');
+ scanDir('ui/src/test/diff_viewer');
scanDir('buildtools/typefaces');
scanDir('buildtools/catapult_trace_viewer');
generateImports('ui/src/tracks', 'all_tracks.ts');
@@ -306,6 +313,10 @@
addTask(cp, [src, pjoin(cfg.outDistDir, 'assets', dst)]);
}
+function copyUiTestArtifactsAssets(src, dst) {
+ addTask(cp, [src, pjoin(cfg.outUiTestArtifactsDir, dst)]);
+}
+
function compileScss() {
const src = pjoin(ROOT_DIR, 'ui/src/assets/perfetto.scss');
const dst = pjoin(cfg.outDistDir, 'perfetto.css');
diff --git a/ui/src/test/diff_viewer/README.md b/ui/src/test/diff_viewer/README.md
new file mode 100644
index 0000000..119dd75
--- /dev/null
+++ b/ui/src/test/diff_viewer/README.md
@@ -0,0 +1,18 @@
+# CI screenshot diff viewer
+
+This directory contains the source of screenshots diff viewer used on Perfetto
+CI. The way it works as follows:
+
+When a screenshot test is failing, the testing code will write a line of the
+form
+
+```
+failed-screenshot.png;failed-screenshot-diff.png
+```
+
+To a file called `report.txt`. Diff viewer is just a static page that uses Fetch
+API to download this file, parse it, and display images in a list of rows.
+
+The page assumes `report.txt` to be present in the same directory, same goes for
+screenshot files. To simplify deployment, the viewer is developed without a
+framework and constructs DOM using `document.createElement` API.
diff --git a/ui/src/test/diff_viewer/index.html b/ui/src/test/diff_viewer/index.html
new file mode 100644
index 0000000..d4742b4
--- /dev/null
+++ b/ui/src/test/diff_viewer/index.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <title>Diff screenshots report</title>
+ <style>
+ .row {
+ display: flex;
+ padding: 1rem;
+ border-radius: .5rem;
+ border: 1px solid black;
+ margin-bottom: 1rem;
+ }
+ .image-wrapper img {
+ max-width: 45vw;
+ }
+ </style>
+</head>
+<body>
+ <div class="container">
+ Loading...
+ </div>
+ <script src="script.js"></script>
+</body>
+</html>
\ No newline at end of file
diff --git a/ui/src/test/diff_viewer/script.js b/ui/src/test/diff_viewer/script.js
new file mode 100644
index 0000000..97a6f64
--- /dev/null
+++ b/ui/src/test/diff_viewer/script.js
@@ -0,0 +1,79 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Helper function to create DOM elements faster: takes a Mithril-style
+// "selector" of the form "tag.class1.class2" and a list of child objects that
+// can be either strings or DOM elements.
+function m(selector, ...children) {
+ const parts = selector.split('.');
+ if (parts.length === 0) {
+ throw new Error(
+ 'Selector passed to element should be of a form tag.class1.class2');
+ }
+
+ const result = document.createElement(parts[0]);
+ for (let i = 1; i < parts.length; i++) {
+ result.classList.add(parts[i]);
+ }
+ for (const child of children) {
+ if (typeof child === 'string') {
+ const childNode = document.createTextNode(child);
+ result.appendChild(childNode);
+ } else {
+ result.appendChild(child);
+ }
+ }
+ return result;
+}
+
+async function loadDiffs() {
+ // report.txt is a text file with a pair of file names on each line, separated
+ // by semicolon. E.g. "screenshot.png;screenshot-diff.png"
+ const report = await fetch('report.txt');
+ const response = await report.text();
+ console.log(response);
+
+ const container = document.querySelector('.container');
+ container.innerHTML = '';
+
+ const lines = response.split('\n');
+ for (const line of lines) {
+ const parts = line.split(';');
+ if (parts.length !== 2) {
+ console.warn(
+ `Malformed line (expected two files separated via semicolon) ${
+ line}!`);
+ continue;
+ }
+
+ const [output, diff] = parts;
+ const outputImage = m('img');
+ outputImage.src = output;
+ const diffImage = m('img');
+ diffImage.src = diff;
+
+ container.appendChild(
+ m('div.row',
+ m('div.cell', output, m('div.image-wrapper', outputImage)),
+ m('div.cell', diff, m('div.image-wrapper', diffImage))));
+ }
+
+ if (lines.length === 0) {
+ container.appendChild(m('div', 'All good!'));
+ }
+}
+
+document.addEventListener('DOMContentLoaded', () => {
+ loadDiffs();
+});
diff --git a/ui/src/test/perfetto_ui_test_helper.ts b/ui/src/test/perfetto_ui_test_helper.ts
index 44e927c..97c9bd4 100644
--- a/ui/src/test/perfetto_ui_test_helper.ts
+++ b/ui/src/test/perfetto_ui_test_helper.ts
@@ -84,7 +84,7 @@
}
export async function compareScreenshots(
- actualFilename: string, expectedFilename: string) {
+ reportPath: string, actualFilename: string, expectedFilename: string) {
if (!fs.existsSync(expectedFilename)) {
throw new Error(
`Could not find ${expectedFilename}. Run wih REBASELINE=1.`);
@@ -102,6 +102,9 @@
if (diff > DIFF_MAX_PIXELS) {
const diffFilename = actualFilename.replace('.png', '-diff.png');
fs.writeFileSync(diffFilename, PNG.sync.write(diffPng));
+ fs.appendFileSync(
+ reportPath,
+ `${path.basename(actualFilename)};${path.basename(diffFilename)}\n`);
fail(`Diff test failed on ${diffFilename}, delta: ${diff} pixels`);
}
return diff;
diff --git a/ui/src/test/ui_integrationtest.ts b/ui/src/test/ui_integrationtest.ts
index 24a6533..4794d8d 100644
--- a/ui/src/test/ui_integrationtest.ts
+++ b/ui/src/test/ui_integrationtest.ts
@@ -28,6 +28,8 @@
declare let global: {__BROWSER__: puppeteer.Browser;};
const browser = assertExists(global.__BROWSER__);
const expectedScreenshotPath = path.join('test', 'data', 'ui-screenshots');
+const tmpDir = path.resolve('./ui-test-artifacts');
+const reportPath = path.join(tmpDir, 'report.txt');
async function getPage(): Promise<puppeteer.Page> {
const pages = (await browser.pages());
@@ -41,6 +43,9 @@
jest.setTimeout(60000);
const page = await getPage();
await page.setViewport({width: 1920, height: 1080});
+
+ // Empty the file with collected screenshot diffs
+ fs.writeFileSync(reportPath, '');
});
// After each test (regardless of nesting) capture a screenshot named after the
@@ -51,10 +56,6 @@
testName = testName.replace(/[^a-z0-9-]/gmi, '_').toLowerCase();
const page = await getPage();
- // cwd() is set to //out/ui when running tests, just create a subdir in there.
- // The CI picks up this directory and uploads to GCS after every failed run.
- const tmpDir = path.resolve('./ui-test-artifacts');
- if (!fs.existsSync(tmpDir)) fs.mkdirSync(tmpDir);
const screenshotName = `ui-${testName}.png`;
const actualFilename = path.join(tmpDir, screenshotName);
const expectedFilename = path.join(expectedScreenshotPath, screenshotName);
@@ -64,7 +65,7 @@
console.log('Saving reference screenshot into', expectedFilename);
fs.copyFileSync(actualFilename, expectedFilename);
} else {
- await compareScreenshots(actualFilename, expectedFilename);
+ await compareScreenshots(reportPath, actualFilename, expectedFilename);
}
});
