blob: 87f5d65e817db62c9c7f57001a088020f5e5796b [file] [log] [blame]
// Copyright 2022 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'ci_successful_test_data.dart';
import 'package:github/github.dart' as github;
import 'package:test/test.dart';
import 'package:auto_submit/validations/ci_successful.dart';
import 'package:auto_submit/model/auto_submit_query_result.dart';
import 'package:auto_submit/validations/validation.dart';
import 'package:auto_submit/configuration/repository_configuration.dart';
import '../utilities/utils.dart';
import '../utilities/mocks.dart';
import '../src/service/fake_config.dart';
import '../src/service/fake_github_service.dart';
import '../src/service/fake_graphql_client.dart';
import '../requests/github_webhook_test_data.dart';
import '../configuration/repository_configuration_data.dart';
void main() {
late CiSuccessful ciSuccessful;
late FakeConfig config;
FakeGithubService githubService = FakeGithubService();
late FakeGraphQLClient githubGraphQLClient;
final MockGitHub gitHub = MockGitHub();
late github.RepositorySlug slug;
late Set<FailureDetail> failures;
const int prNumber = 1;
List<ContextNode> getContextNodeListFromJson(final String repositoryStatuses) {
final List<ContextNode> contextNodeList = [];
final Map<String, dynamic> contextNodeMap = jsonDecode(repositoryStatuses) as Map<String, dynamic>;
final dynamic statuses = contextNodeMap['statuses'];
for (Map<String, dynamic> map in statuses) {
contextNodeList.add(ContextNode.fromJson(map));
}
return contextNodeList;
}
void convertContextNodeStatuses(List<ContextNode> contextNodeList) {
for (ContextNode contextNode in contextNodeList) {
contextNode.state = contextNode.state!.toUpperCase();
}
}
/// Setup objects needed across test groups.
setUp(() {
githubGraphQLClient = FakeGraphQLClient();
config = FakeConfig(githubService: githubService, githubGraphQLClient: githubGraphQLClient, githubClient: gitHub);
ciSuccessful = CiSuccessful(config: config);
slug = github.RepositorySlug('octocat', 'flutter');
failures = <FailureDetail>{};
config.repositoryConfigurationMock = RepositoryConfiguration.fromYaml(sampleConfigNoOverride);
});
group('validateCheckRuns', () {
test('ValidateCheckRuns no failures for skipped conclusion.', () {
githubService.checkRunsData = skippedCheckRunsMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isTrue);
expect(failures, isEmpty);
});
});
test('ValidateCheckRuns no failures for successful conclusion.', () {
githubService.checkRunsData = checkRunsMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isTrue);
expect(failures, isEmpty);
});
});
test('ValidateCheckRuns no failure for status completed and neutral conclusion.', () {
githubService.checkRunsData = neutralCheckRunsMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isTrue);
expect(failures, isEmpty);
});
});
test('ValidateCheckRuns failure detected on status completed no neutral conclusion.', () {
githubService.checkRunsData = failedCheckRunsMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isFalse);
expect(failures, isNotEmpty);
expect(failures.length, 1);
});
});
test('ValidateCheckRuns succes with multiple successful check runs.', () {
githubService.checkRunsData = multipleCheckRunsMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isTrue);
expect(failures, isEmpty);
});
});
test('ValidateCheckRuns failed with multiple check runs.', () {
githubService.checkRunsData = multipleCheckRunsWithFailureMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isFalse);
expect(failures, isNotEmpty);
expect(failures.length, 1);
});
});
test('ValidateCheckRuns allSucces false but no failures recorded.', () {
/// This test just checks that a checkRun that has not yet completed and
/// does not cause failure is a candidate to be temporarily ignored.
githubService.checkRunsData = inprogressAndNotFailedCheckRunMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = true;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isFalse);
expect(failures, isEmpty);
});
});
test('ValidateCheckRuns allSuccess false is preserved.', () {
githubService.checkRunsData = multipleCheckRunsWithFailureMock;
final Future<List<github.CheckRun>> checkRunFuture = githubService.getCheckRuns(slug, 'ref');
const bool allSuccess = false;
checkRunFuture.then((checkRuns) {
expect(ciSuccessful.validateCheckRuns(slug, prNumber, checkRuns, failures, allSuccess), isFalse);
expect(failures, isNotEmpty);
expect(failures.length, 1);
});
});
});
group('validateStatuses', () {
test('Validate successful statuses show as successful.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesMock);
const bool allSuccess = true;
final Author author = Author(login: 'ricardoamador');
/// The status must be uppercase as the original code is expecting this.
convertContextNodeStatuses(contextNodeList);
expect(ciSuccessful.validateStatuses(slug, prNumber, author, [], contextNodeList, failures, allSuccess), isTrue);
expect(failures, isEmpty);
});
test('Validate statuses that are not successful but do not cause failure.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(failedAuthorsStatusesMock);
const bool allSuccess = true;
final Author author = Author(login: 'ricardoamador');
final List<String> labelNames = [];
labelNames.add('warning: land on red to fix tree breakage');
labelNames.add('Other label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isTrue,
);
expect(failures, isEmpty);
});
test('Validate failure statuses do not cause failure with not in authors control.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(failedAuthorsStatusesMock);
const bool allSuccess = true;
final Author author = Author(login: 'ricardoamador');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isFalse,
);
expect(failures, isEmpty);
});
test('Validate failure statuses cause failures with not in authors control.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(failedNonAuthorsStatusesMock);
const bool allSuccess = true;
final Author author = Author(login: 'ricardoamador');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isFalse,
);
expect(failures, isNotEmpty);
expect(failures.length, 2);
});
test('Validate failure statuses cause failures and preserves false allSuccess.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(failedNonAuthorsStatusesMock);
const bool allSuccess = false;
final Author author = Author(login: 'ricardoamador');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isFalse,
);
expect(failures, isNotEmpty);
expect(failures.length, 2);
});
test('Validate flutter-gold is not checked for engine auto roller pull requests.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesWithGoldMock);
const bool allSuccess = true;
final Author author = Author(login: 'skia-flutter-autoroll');
slug = github.RepositorySlug('flutter', 'engine');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isTrue,
);
expect(failures, isEmpty);
expect(failures.length, 0);
});
test('Validate flutter-gold is not checked even if failing for engine auto roller pull requests.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesWithFailedGoldMock);
const bool allSuccess = true;
final Author author = Author(login: 'skia-flutter-autoroll');
slug = github.RepositorySlug('flutter', 'engine');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isTrue,
);
expect(failures, isEmpty);
expect(failures.length, 0);
});
test('Validate flutter-gold is checked for non engine auto roller pull requests.', () {
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesWithFailedGoldMock);
const bool allSuccess = true;
final Author author = Author(login: 'ricardoamador');
slug = github.RepositorySlug('flutter', 'engine');
final List<String> labelNames = [];
labelNames.add('Compelling label');
labelNames.add('Another Compelling label');
convertContextNodeStatuses(contextNodeList);
expect(
ciSuccessful.validateStatuses(slug, prNumber, author, labelNames, contextNodeList, failures, allSuccess),
isFalse,
);
expect(failures, isNotEmpty);
expect(failures.length, 1);
});
});
group('treeStatusCheck', () {
test('Validate tree status is set contains slug.', () {
slug = github.RepositorySlug('flutter', 'flutter');
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesMock);
expect(contextNodeList.isEmpty, false);
/// The status must be uppercase as the original code is expecting this.
convertContextNodeStatuses(contextNodeList);
final bool treeStatusFlag = ciSuccessful.treeStatusCheck(slug, prNumber, contextNodeList);
expect(treeStatusFlag, true);
});
test('Validate tree status is set does not contain slug.', () {
slug = github.RepositorySlug('flutter', 'infra');
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesMock);
expect(contextNodeList.isEmpty, false);
/// The status must be uppercase as the original code is expecting this.
convertContextNodeStatuses(contextNodeList);
final bool treeStatusFlag = ciSuccessful.treeStatusCheck(slug, prNumber, contextNodeList);
expect(treeStatusFlag, true);
});
test('Validate tree status is set but context does not match slug.', () {
slug = github.RepositorySlug('flutter', 'flutter');
final List<ContextNode> contextNodeList = getContextNodeListFromJson(repositoryStatusesNonLuciFlutterMock);
/// The status must be uppercase as the original code is expecting this.
convertContextNodeStatuses(contextNodeList);
final bool treeStatusFlag = ciSuccessful.treeStatusCheck(slug, prNumber, contextNodeList);
expect(treeStatusFlag, false);
});
});
group('validate', () {
test('Commit has a null status, no statuses to verify.', () {
final Map<String, dynamic> queryResultJsonDecode =
jsonDecode(nullStatusCommitRepositoryJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final Commit commit = pr.commits!.nodes!.single.commit!;
expect(commit, isNotNull);
expect(commit.status, isNull);
final github.PullRequest npr = generatePullRequest();
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// fails because in this case there is only a single fail status
expect(false, value.result);
// Remove label.
expect(value.action, Action.IGNORE_TEMPORARILY);
expect(value.message, 'Hold to wait for the tree status ready.');
});
});
test('Commit has no statuses to verify.', () {
final Map<String, dynamic> queryResultJsonDecode = jsonDecode(noStatusInCommitJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final github.PullRequest npr = generatePullRequest();
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// fails because in this case there is only a single fail status
expect(false, value.result);
// Remove label.
expect(value.action, Action.IGNORE_TEMPORARILY);
expect(value.message, 'Hold to wait for the tree status ready.');
});
});
// When branch is default we need to wait for the tree status if it is not
// present.
test('Commit has no statuses to verify.', () {
final Map<String, dynamic> queryResultJsonDecode = jsonDecode(noStatusInCommitJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final github.PullRequest npr = generatePullRequest();
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// fails because in this case there is only a single fail status
expect(false, value.result);
// Remove label.
expect(value.action, Action.IGNORE_TEMPORARILY);
expect(value.message, 'Hold to wait for the tree status ready.');
});
});
// Test for when the base branch is not default, we should not check the
// tree status as it does not apply.
test('Commit has no statuses to verify and base branch is not default.', () {
final Map<String, dynamic> queryResultJsonDecode = jsonDecode(noStatusInCommitJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final github.PullRequest npr = generatePullRequest(baseRef: 'test_feature');
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
expect(true, value.result);
// Remove label.
expect(value.action, Action.REMOVE_LABEL);
expect(value.message, isEmpty);
});
});
test('Commit has statuses to verify, action remove label, no message.', () {
final Map<String, dynamic> queryResultJsonDecode =
jsonDecode(nonNullStatusSUCCESSCommitRepositoryJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final Commit commit = pr.commits!.nodes!.single.commit!;
expect(commit, isNotNull);
expect(commit.status, isNotNull);
final github.PullRequest npr = generatePullRequest();
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// No failure.
expect(value.result, isTrue);
// Remove label.
expect((value.action == Action.REMOVE_LABEL), isTrue);
expect(value.message, isEmpty);
});
});
test('Commit has statuses to verify, action ignore failure, no message.', () {
final Map<String, dynamic> queryResultJsonDecode =
jsonDecode(nonNullStatusFAILURECommitRepositoryJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final Commit commit = pr.commits!.nodes!.single.commit!;
expect(commit, isNotNull);
expect(commit.status, isNotNull);
final github.PullRequest npr = generatePullRequest(labelName: 'warning: land on red to fix tree breakage');
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// No failure.
expect(value.result, isTrue);
// Remove label.
expect((value.action == Action.IGNORE_FAILURE), isTrue);
expect(value.message, isEmpty);
});
});
test('Commit has statuses to verify, action failure, no message.', () {
final Map<String, dynamic> queryResultJsonDecode =
jsonDecode(nonNullStatusFAILURECommitRepositoryJson) as Map<String, dynamic>;
final QueryResult queryResult = QueryResult.fromJson(queryResultJsonDecode);
expect(queryResult, isNotNull);
final PullRequest pr = queryResult.repository!.pullRequest!;
expect(pr, isNotNull);
final Commit commit = pr.commits!.nodes!.single.commit!;
expect(commit, isNotNull);
expect(commit.status, isNotNull);
final github.PullRequest npr = generatePullRequest();
githubService.checkRunsData = checkRunsMock;
ciSuccessful.validate(queryResult, npr).then((value) {
// No failure.
expect(false, value.result);
// Remove label.
expect((value.action == Action.IGNORE_TEMPORARILY), isTrue);
expect(value.message, isEmpty);
});
});
});
group('Validate empty message is not returned.', () {
setUp(() {
githubService = FakeGithubService(client: MockGitHub());
config = FakeConfig(githubService: githubService);
ciSuccessful = CiSuccessful(config: config);
slug = github.RepositorySlug('flutter', 'cocoon');
config.repositoryConfigurationMock = RepositoryConfiguration.fromYaml(sampleConfigNoOverride);
});
test('returns correct message when validation fails', () async {
final PullRequestHelper flutterRequest = PullRequestHelper(
prNumber: 0,
lastCommitHash: oid,
reviews: <PullRequestReviewHelper>[],
);
githubService.checkRunsData = failedCheckRunsMock;
final github.PullRequest pullRequest = generatePullRequest(prNumber: 0, repoName: slug.name);
final QueryResult queryResult = createQueryResult(flutterRequest);
final ValidationResult validationResult = await ciSuccessful.validate(queryResult, pullRequest);
expect(validationResult.result, false);
expect(
validationResult.message,
'- The status or check suite [failed_checkrun](https://example.com) has failed. Please fix the issues identified (or deflake) before re-applying this label.\n',
);
});
});
}