dev/devicelab/test/run_test.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';

import 'package:flutter_devicelab/framework/utils.dart' show rm;
import 'package:path/path.dart' as path;
import 'package:process/process.dart';

import 'common.dart';

void main() {
  const ProcessManager processManager = LocalProcessManager();
  final String dart = path.absolute(
    path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'),
  );

  group('run.dart script', () {
    // The tasks here refer to files in ../bin/tasks/*.dart

    Future<ProcessResult> runScript(
      List<String> taskNames, [
      List<String> otherArgs = const <String>[],
    ]) async {
      final ProcessResult scriptProcess = processManager.runSync(<String>[
        dart,
        'bin/run.dart',
        '--no-terminate-stray-dart-processes',
        ...otherArgs,
        for (final String testName in taskNames) ...<String>['-t', testName],
      ]);
      return scriptProcess;
    }

    Future<void> expectScriptResult(
      List<String> taskNames,
      int expectedExitCode, {
      String? deviceId,
    }) async {
      final ProcessResult result = await runScript(taskNames, <String>[
        if (deviceId != null) ...<String>['-d', deviceId],
      ]);
      expect(
        result.exitCode,
        expectedExitCode,
        reason:
            '[ stderr from test process ]\n'
            '\n'
            '${result.stderr}\n'
            '\n'
            '[ end of stderr ]\n'
            '\n'
            '[ stdout from test process ]\n'
            '\n'
            '${result.stdout}\n'
            '\n'
            '[ end of stdout ]',
      );
    }

    test('exits with code 0 when succeeds', () async {
      await expectScriptResult(<String>['smoke_test_success'], 0);
    });

    test('rejects invalid file paths', () async {
      await expectScriptResult(<String>['lib/framework/adb.dart'], 1);
    });

    test('exits with code 1 when task throws', () async {
      await expectScriptResult(<String>['smoke_test_throws'], 1);
    });

    test('exits with code 1 when fails', () async {
      await expectScriptResult(<String>['smoke_test_failure'], 1);
    });

    test(
      'prints a message after a few seconds when failing to connect (this test takes >10s)',
      () async {
        final Process process = await processManager.start(<String>[
          dart,
          'bin/run.dart',
          '--no-terminate-stray-dart-processes',
          '-t',
          'smoke_test_setup_failure',
        ]);

        // If this test fails, the reason is usually buried in stderr.
        final Stream<String> stderr = process.stderr.transform(utf8.decoder);
        stderr.listen(printOnFailure);

        final Stream<String> stdout = process.stdout.transform(utf8.decoder);
        await expectLater(
          stdout,
          emitsThrough(
            contains('VM service still not ready. It is possible the target has failed'),
          ),
        );
        expect(process.kill(), isTrue);
      },
      timeout: const Timeout(Duration(seconds: 45)),
    ); // Standard 30 is flaky because this is a long running test, https://github.com/flutter/flutter/issues/156456

    test('exits with code 1 when results are mixed', () async {
      await expectScriptResult(<String>['smoke_test_failure', 'smoke_test_success'], 1);
    });

    test('exits with code 0 when provided a valid device ID', () async {
      await expectScriptResult(<String>['smoke_test_device'], 0, deviceId: 'FAKE');
    });

    test('exits with code 1 when provided a bad device ID', () async {
      await expectScriptResult(<String>['smoke_test_device'], 1, deviceId: 'THIS_IS_NOT_VALID');
    });

    test('runs A/B test', () async {
      final Directory tempDirectory = Directory.systemTemp.createTempSync(
        'flutter_devicelab_ab_test.',
      );
      final abResultsFile = File(path.join(tempDirectory.path, 'test_results.json'));

      expect(abResultsFile.existsSync(), isFalse);

      final ProcessResult result = await runScript(
        <String>['smoke_test_success'],
        <String>[
          '--ab=2',
          '--local-engine=host_debug_unopt',
          '--local-engine-host=host_debug_unopt',
          '--ab-result-file',
          abResultsFile.path,
        ],
      );
      expect(result.exitCode, 0);

      var sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════'
          : 'A/B results so far';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════'
          : 'Raw results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'metric1:\n'
          '  A:\t42.00\t42.00\t\n'
          '  B:\t42.00\t42.00\t\n'
          'metric2:\n'
          '  A:\t123.00\t123.00\t\n'
          '  B:\t123.00\t123.00\t\n',
        ),
      );

      sectionHeader = !Platform.isWindows
          ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════'
          : 'Final A/B results';
      expect(
        result.stdout,
        contains(
          '$sectionHeader\n'
          '\n'
          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
        ),
      );

      expect(abResultsFile.existsSync(), isTrue);
      rm(tempDirectory, recursive: true);
    });

    test('fails to upload results to Cocoon if flags given', () async {
      // CocoonClient will fail to find test-file, and will not send any http requests.
      final ProcessResult result = await runScript(
        <String>['smoke_test_success'],
        <String>['--service-account-file=test-file', '--task-key=task123'],
      );
      expect(result.exitCode, 1);
    });

    test('passes build mode to the test', () async {
      final ProcessResult result = await runScript(
        <String>['build_mode_test'],
        <String>['--build-mode=debug'],
      );
      expect(result.exitCode, 0);
      expect(result.stdout, contains('buildMode: debug'));
    });
  });
}