blob: f1574435d77d0b2d03f0f315c7e6c3f0fc1f618b [file] [log] [blame]
// Copyright 2019 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:typed_data';
import 'package:appengine/appengine.dart';
import 'package:cocoon_service/src/service/datastore.dart';
import 'package:corsac_jwt/corsac_jwt.dart';
import 'package:gcloud/db.dart';
import 'package:gcloud/service_scope.dart' as ss;
import 'package:github/github.dart' as gh;
import 'package:googleapis/bigquery/v2.dart';
import 'package:graphql/client.dart';
import 'package:http/http.dart' as http;
import 'package:meta/meta.dart';
import 'package:retry/retry.dart';
import '../../cocoon_service.dart';
import '../model/appengine/branch.dart';
import '../model/appengine/cocoon_config.dart';
import '../model/appengine/key_helper.dart';
import 'access_client_provider.dart';
import 'bigquery.dart';
import 'github_service.dart';
import 'logging.dart';
/// Name of the default git branch.
const String kDefaultBranchName = 'master';
class Config {
Config(this._db, this._cache);
final DatastoreDB _db;
final CacheService _cache;
/// List of Github presubmit supported repos.
///
/// This adds support for the `waiting for tree to go green label` to the repo.
///
/// Relies on the GitHub Checks API being enabled for this repo.
Set<gh.RepositorySlug> get supportedRepos => <gh.RepositorySlug>{
cocoonSlug,
engineSlug,
flutterSlug,
packagesSlug,
};
/// List of guaranteed scheduling Github repos.
static Set<gh.RepositorySlug> get guaranteedSchedulingRepos => <gh.RepositorySlug>{
engineSlug,
packagesSlug,
};
/// List of Github postsubmit supported repos.
///
/// This adds support for check runs to the repo.
Set<gh.RepositorySlug> get postsubmitSupportedRepos => <gh.RepositorySlug>{
packagesSlug,
};
/// GitHub repositories that use CI status to determine if pull requests can be submitted.
static Set<gh.RepositorySlug> reposWithTreeStatus = <gh.RepositorySlug>{
engineSlug,
flutterSlug,
};
/// The tip of tree branch for [slug].
static String defaultBranch(gh.RepositorySlug slug) {
final Map<gh.RepositorySlug, String> defaultBranches = <gh.RepositorySlug, String>{
cocoonSlug: 'main',
flutterSlug: 'master',
engineSlug: 'main',
packagesSlug: 'main',
recipesSlug: 'main',
};
return defaultBranches[slug] ?? kDefaultBranchName;
}
// The name of the bot that generates automated revert requests.
String get autosubmitBot => 'auto-submit[bot]';
// The name of the label that the bot uses to identify the automatically
// created pull request.
static const String revertOfLabel = 'revert of';
/// Memorystore subcache name to store [CocoonConfig] values in.
static const String configCacheName = 'config';
/// Default properties when rerunning a prod build.
static const Map<String, Object> defaultProperties = <String, Object>{'force_upload': true};
@visibleForTesting
static const Duration configCacheTtl = Duration(hours: 12);
Logging get loggingService => ss.lookup(#appengine.logging) as Logging;
Future<Iterable<Branch>> getBranches(gh.RepositorySlug slug) async {
final DatastoreService datastore = DatastoreService(db, defaultMaxEntityGroups);
final List<Branch> branches = await (datastore.queryBranches().toList());
return branches.where((Branch branch) => branch.slug == slug);
}
Future<List<String>> _getReleaseAccounts() async {
final String releaseAccountsConcat = await _getSingleValue('ReleaseAccounts');
return releaseAccountsConcat.split(',');
}
Future<String> _getSingleValue(String id) async {
final Uint8List? cacheValue = await _cache.getOrCreate(
configCacheName,
id,
createFn: () => _getValueFromDatastore(id),
ttl: configCacheTtl,
);
return String.fromCharCodes(cacheValue!);
}
Future<Uint8List> _getValueFromDatastore(String id) async {
final CocoonConfig cocoonConfig = CocoonConfig()
..id = id
..parentKey = _db.emptyKey;
final CocoonConfig result = await _db.lookupValue<CocoonConfig>(cocoonConfig.key);
return Uint8List.fromList(result.value.codeUnits);
}
// GitHub App properties.
Future<String> get githubPrivateKey => _getSingleValue('githubapp_private_pem');
Future<String> get overrideTreeStatusLabel => _getSingleValue('override_tree_status_label');
Future<String> get githubPublicKey => _getSingleValue('githubapp_public_pem');
Future<String> get githubAppId => _getSingleValue('githubapp_id');
Future<Map<String, dynamic>> get githubAppInstallations async {
final String installations = await _getSingleValue('githubapp_installations');
return jsonDecode(installations) as Map<String, dynamic>;
}
// Default recipe bundle used when the PR's base branch name does not exist in
// the recipes GoB project.
String get defaultRecipeBundleRef => 'refs/heads/main';
DatastoreDB get db => _db;
/// Size of the shards to send to buildBucket when scheduling builds.
int get schedulingShardSize => 5;
/// Batch size of builds to schedule in each swarming request.
int get batchSize => 5;
/// Upper limit of targets to be backfilled in API call.
///
/// For example, if we have 200 available targets found to be backfilled,
/// only `backfillerTargetLimit` will be scheduled whereas others wait for
/// the next API call.
int get backfillerTargetLimit => 50;
/// Upper limit of commit rows to be backfilled in API call.
///
/// This limits the number of commits to be checked to backfill. When bots
/// are idle, we hope to scan as many commit rows as possible.
int get backfillerCommitLimit => 50;
/// Upper limit of issue/PRs allowed each API call.
///
/// GitHub enforces a secondary rate limit on frequency API calls. This causes
/// our API failure when many issues/PRs are created in a short time.
int get issueAndPRLimit => 2;
/// Max retries when scheduling builds.
static const RetryOptions schedulerRetry = RetryOptions(maxAttempts: 3);
/// List of GitHub accounts related to releases.
Future<List<String>> get releaseAccounts => _getReleaseAccounts();
Future<String> get oauthClientId => _getSingleValue('OAuthClientId');
/// Webhook secret for the "Flutter Roll on Borg" GitHub App.
Future<String> get frobWebhookKey => _getSingleValue('FrobWebhookKey');
Future<String> get githubOAuthToken => _getSingleValue('GitHubPRToken');
String get wrongBaseBranchPullRequestMessage => 'This pull request was opened against a branch other than '
'_{{default_branch}}_. Since Flutter pull requests should not '
'normally be opened against branches other than {{default_branch}}, I '
'have changed the base to {{default_branch}}. If this was intended, you '
'may modify the base back to {{target_branch}}. See the [Release Process]'
'(https://github.com/flutter/flutter/wiki/Release-process) for information '
'about how other branches get updated.\n\n'
'__Reviewers__: Use caution before merging pull requests to branches other '
'than {{default_branch}}, unless this is an intentional hotfix/cherrypick.';
String wrongHeadBranchPullRequestMessage(String branch) =>
'This pull request is trying merge the branch $branch, which is the name '
'of a release branch. This is usually a mistake. See '
'[Tree Hygiene](https://github.com/flutter/flutter/wiki/Tree-hygiene) '
'for detailed instructions on how to contribute to the Flutter project. '
'In particular, ensure that before you start coding, you create your '
'feature branch off of _${kDefaultBranchName}_.\n\n'
'This PR has been closed. If you are sure you want to merge $branch, you '
'may re-open this issue.';
String get releaseBranchPullRequestMessage => 'This pull request was opened '
'from and to a release candidate branch. This should only be done as part '
'of the official [Flutter release process]'
'(https://github.com/flutter/flutter/wiki/Release-process). If you are '
'attempting to make a regular contribution to the Flutter project, please '
'close this PR and follow the instructions at [Tree Hygiene]'
'(https://github.com/flutter/flutter/wiki/Tree-hygiene) for detailed '
'instructions on contributing to Flutter.\n\n'
'__Reviewers__: Use caution before merging pull requests to release '
'branches. Ensure the proper procedure has been followed.';
Future<String> get webhookKey => _getSingleValue('WebhookKey');
String get mergeConflictPullRequestMessage => 'This pull request is not '
'mergeable in its current state, likely because of a merge conflict. '
'Pre-submit CI jobs were not triggered. Pushing a new commit to this '
'branch that resolves the issue will result in pre-submit jobs being '
'scheduled.';
String get missingTestsPullRequestMessage => 'It looks like this pull '
'request may not have tests. Please make sure to add tests before merging. '
'If you need '
'[an exemption](https://github.com/flutter/flutter/wiki/Tree-hygiene#tests) '
'to this rule, contact Hixie or stuartmorgan on the #hackers '
'channel in [Chat](https://github.com/flutter/flutter/wiki/Chat) '
'(don\'t just cc them here, they won\'t see it! Use Discord!).'
'\n\n'
'If you are not sure if you need tests, consider this rule of thumb: '
'the purpose of a test is to make sure someone doesn\'t accidentally '
'revert the fix. Ask yourself, **is there anything in your PR that you '
'feel it is important we not accidentally revert back to how it was '
'before your fix?**'
'\n\n'
'__Reviewers__: Read the [Tree Hygiene page]'
'(https://github.com/flutter/flutter/wiki/Tree-hygiene#how-to-review-code) '
'and make sure this patch meets those guidelines before LGTMing.';
String get flutterGoldPending => 'Waiting for all other checks to be successful before querying Gold.';
String get flutterGoldSuccess => 'All golden file tests have passed.';
String get flutterGoldChanges => 'Image changes have been found for '
'this pull request.';
String get flutterGoldStalePR => 'This pull request executed golden file '
'tests, but it has not been updated in a while (20+ days). Test results from '
'Gold expire after as many days, so this pull request will need to be '
'updated with a fresh commit in order to get results from Gold.';
String get flutterGoldDraftChange => 'This pull request has been changed to a '
'draft. The currently pending flutter-gold status will not be able '
'to resolve until a new commit is pushed or the change is marked ready for '
'review again.';
String flutterGoldInitialAlert(String url) => 'Golden file changes have been found for this pull '
'request. Click [here to view and triage]($url) '
'(e.g. because this is an intentional change).\n\n'
'If you are still iterating on this change and are not ready to '
'resolve the images on the Flutter Gold dashboard, consider marking this PR '
'as a draft pull request above. You will still be able to view image results '
'on the dashboard, commenting will be silenced, and the check will not try to resolve itself until '
'marked ready for review.\n\n';
String flutterGoldFollowUpAlert(String url) => 'Golden file changes are available for triage from new commit, '
'Click [here to view]($url).\n\n';
String flutterGoldAlertConstant(gh.RepositorySlug slug) {
if (slug == Config.flutterSlug) {
return '\n\nFor more guidance, visit '
'[Writing a golden file test for `package:flutter`](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).\n\n'
'__Reviewers__: Read the [Tree Hygiene page](https://github.com/flutter/flutter/wiki/Tree-hygiene#how-to-review-code) '
'and make sure this patch meets those guidelines before LGTMing.\n\n';
}
return '';
}
String flutterGoldCommentID(gh.PullRequest pr) =>
'_Changes reported for pull request #${pr.number} at sha ${pr.head!.sha}_\n\n';
/// Post submit service account email used by LUCI swarming tasks.
static const String luciProdAccount = 'flutter-prod-builder@chops-service-accounts.iam.gserviceaccount.com';
/// Internal Google service account used to surface FRoB results.
static const String frobAccount = 'flutter-roll-on-borg@flutter-roll-on-borg.google.com.iam.gserviceaccount.com';
/// Service accounts used for PubSub messages.
static const Set<String> allowedPubsubServiceAccounts = <String>{
'flutter-devicelab@flutter-dashboard.iam.gserviceaccount.com',
'flutter-dashboard@appspot.gserviceaccount.com',
};
int get maxTaskRetries => 2;
/// Max retries for Luci builder with infra failure.
int get maxLuciTaskRetries => 2;
/// The default number of commit shown in flutter build dashboard.
int get commitNumber => 30;
KeyHelper get keyHelper => KeyHelper(applicationContext: context.applicationContext);
// Default number of commits to return for benchmark dashboard.
int /*!*/ get maxRecords => 50;
// Delay between consecutive GitHub deflake request calls.
Duration get githubRequestDelay => const Duration(seconds: 1);
// Repository status context for github status.
String get flutterBuild => 'flutter-build';
// Repository status description for github status.
String get flutterBuildDescription => 'Tree is currently broken. Please do not merge this '
'PR unless it contains a fix for the tree.';
static gh.RepositorySlug get cocoonSlug => gh.RepositorySlug('flutter', 'cocoon');
static gh.RepositorySlug get engineSlug => gh.RepositorySlug('flutter', 'engine');
static gh.RepositorySlug get flutterSlug => gh.RepositorySlug('flutter', 'flutter');
static gh.RepositorySlug get packagesSlug => gh.RepositorySlug('flutter', 'packages');
/// Flutter recipes is hosted on Gerrit instead of GitHub.
static gh.RepositorySlug get recipesSlug => gh.RepositorySlug('flutter', 'recipes');
String get waitingForTreeToGoGreenLabelName => 'waiting for tree to go green';
/// The names of autoroller accounts for the repositories.
///
/// These accounts should not need reviews before merging. See
/// https://github.com/flutter/flutter/wiki/Autorollers
Set<String> get rollerAccounts => const <String>{
'skia-flutter-autoroll',
'engine-flutter-autoroll',
'dependabot',
'dependabot[bot]',
};
Future<String> generateJsonWebToken() async {
final String privateKey = await githubPrivateKey;
final String publicKey = await githubPublicKey;
final JWTBuilder builder = JWTBuilder();
final DateTime now = DateTime.now();
builder
..issuer = await githubAppId
..issuedAt = now
..expiresAt = now.add(const Duration(minutes: 10));
final JWTRsaSha256Signer signer = JWTRsaSha256Signer(privateKey: privateKey, publicKey: publicKey);
final JWT signedToken = builder.getSignedToken(signer);
return signedToken.toString();
}
Future<String> generateGithubToken(gh.RepositorySlug slug) async {
// GitHub's secondary rate limits are run into very frequently when making auth tokens.
final Uint8List? cacheValue = await _cache.getOrCreateWithLocking(
configCacheName,
'githubToken-${slug.fullName}',
createFn: () => _generateGithubToken(slug),
// Tokens are minted for 10 minutes
ttl: const Duration(minutes: 8),
);
return String.fromCharCodes(cacheValue!);
}
Future<Uint8List> _generateGithubToken(gh.RepositorySlug slug) async {
final Map<String, dynamic> appInstallations = await githubAppInstallations;
final String? appInstallation = appInstallations[slug.fullName]['installation_id'] as String?;
final String jsonWebToken = await generateJsonWebToken();
final Map<String, String> headers = <String, String>{
'Authorization': 'Bearer $jsonWebToken',
'Accept': 'application/vnd.github.machine-man-preview+json',
};
final Uri githubAccessTokensUri = Uri.https('api.github.com', 'app/installations/$appInstallation/access_tokens');
final http.Response response = await http.post(githubAccessTokensUri, headers: headers);
final Map<String, dynamic> jsonBody = jsonDecode(response.body) as Map<String, dynamic>;
if (jsonBody.containsKey('token') == false) {
log.warning(response.body);
throw Exception('generateGitHubToken failed to get token from Github for repo=${slug.fullName}');
}
final String token = jsonBody['token'] as String;
log.fine('Generated a new GitHub token for ${slug.fullName}');
return Uint8List.fromList(token.codeUnits);
}
Future<gh.GitHub> createGitHubClient({gh.PullRequest? pullRequest, gh.RepositorySlug? slug}) async {
slug ??= pullRequest!.base!.repo!.slug();
final String githubToken = await generateGithubToken(slug);
return createGitHubClientWithToken(githubToken);
}
gh.GitHub createGitHubClientWithToken(String token) {
return gh.GitHub(auth: gh.Authentication.withToken(token));
}
Future<GraphQLClient> createGitHubGraphQLClient() async {
final HttpLink httpLink = HttpLink(
'https://api.github.com/graphql',
defaultHeaders: <String, String>{
'Accept': 'application/vnd.github.antiope-preview+json',
},
);
final String token = await githubOAuthToken;
final AuthLink authLink = AuthLink(
getToken: () async => 'Bearer $token',
);
return GraphQLClient(
cache: GraphQLCache(),
link: authLink.concat(httpLink),
);
}
Future<BigqueryService> createBigQueryService() async {
final AccessClientProvider accessClientProvider = AccessClientProvider();
return BigqueryService(accessClientProvider);
}
Future<TabledataResource> createTabledataResourceApi() async {
return (await createBigQueryService()).defaultTabledata();
}
/// Default GitHub service when the repository does not matter.
///
/// Internally uses the framework repo for OAuth.
Future<GithubService> createDefaultGitHubService() async {
return createGithubService(flutterSlug);
}
Future<GithubService> createGithubService(gh.RepositorySlug slug) async {
final gh.GitHub github = await createGitHubClient(slug: slug);
return GithubService(github);
}
GithubService createGithubServiceWithToken(String token) {
final gh.GitHub github = createGitHubClientWithToken(token);
return GithubService(github);
}
}