Create /api/get-branches to support frontend listing branches (#682)
* create get-branches API
* put common scripts to utils
* comments udpate
diff --git a/app_dart/bin/server.dart b/app_dart/bin/server.dart
index b7798e2..081bf0c 100644
--- a/app_dart/bin/server.dart
+++ b/app_dart/bin/server.dart
@@ -69,6 +69,12 @@
delegate: GetStatus(config),
),
'/api/public/get-timeseries-history': GetTimeSeriesHistory(config),
+ '/api/public/get-branches': CacheRequestHandler<Body>(
+ cache: cache,
+ config: config,
+ delegate: GetBranches(config),
+ ttl: const Duration(minutes: 15),
+ ),
};
return await runAppEngine((HttpRequest request) async {
diff --git a/app_dart/lib/cocoon_service.dart b/app_dart/lib/cocoon_service.dart
index 0570a34..33f08b7 100644
--- a/app_dart/lib/cocoon_service.dart
+++ b/app_dart/lib/cocoon_service.dart
@@ -9,6 +9,7 @@
export 'src/request_handlers/create_agent.dart';
export 'src/request_handlers/get_authentication_status.dart';
export 'src/request_handlers/get_benchmarks.dart';
+export 'src/request_handlers/get_branches.dart';
export 'src/request_handlers/get_build_status.dart';
export 'src/request_handlers/get_log.dart';
export 'src/request_handlers/get_status.dart';
diff --git a/app_dart/lib/src/datastore/cocoon_config.dart b/app_dart/lib/src/datastore/cocoon_config.dart
index af7f234..d739201 100644
--- a/app_dart/lib/src/datastore/cocoon_config.dart
+++ b/app_dart/lib/src/datastore/cocoon_config.dart
@@ -126,6 +126,8 @@
String get cqLabelName => 'CQ+1';
+ RepositorySlug get flutterSlug => const RepositorySlug('flutter', 'flutter');
+
String get waitingForTreeToGoGreenLabelName => 'waiting for tree to go green';
Future<ServiceAccountInfo> get deviceLabServiceAccount async {
diff --git a/app_dart/lib/src/request_handlers/get_branches.dart b/app_dart/lib/src/request_handlers/get_branches.dart
new file mode 100644
index 0000000..2f01c2f
--- /dev/null
+++ b/app_dart/lib/src/request_handlers/get_branches.dart
@@ -0,0 +1,51 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:github/server.dart';
+import 'package:meta/meta.dart';
+
+import '../datastore/cocoon_config.dart';
+import '../foundation/providers.dart';
+import '../foundation/typedefs.dart';
+import '../request_handlers/utils.dart';
+import '../request_handling/body.dart';
+import '../request_handling/request_handler.dart';
+
+/// Queries GitHub for the list of all available branches on
+/// [config.flutterSlug] repo, and returns list of branches
+/// that match pre-defined branch regular expressions.
+@immutable
+class GetBranches extends RequestHandler<Body> {
+ const GetBranches(
+ Config config, {
+ @visibleForTesting
+ this.branchHttpClientProvider = Providers.freshHttpClient,
+ @visibleForTesting this.gitHubBackoffCalculator = twoSecondLinearBackoff,
+ }) : assert(branchHttpClientProvider != null),
+ assert(gitHubBackoffCalculator != null),
+ super(config: config);
+
+ final HttpClientProvider branchHttpClientProvider;
+ final GitHubBackoffCalculator gitHubBackoffCalculator;
+
+ @override
+ Future<Body> get() async {
+ final GitHub github = await config.createGitHubClient();
+ final RepositorySlug slug = config.flutterSlug;
+ final Stream<Branch> branchList = github.repositories.listBranches(slug);
+ final List<String> regExps = await loadBranchRegExps(
+ branchHttpClientProvider, log, gitHubBackoffCalculator);
+ final List<String> branches = <String>[];
+
+ await for (Branch branch in branchList) {
+ if (regExps
+ .any((String regExp) => RegExp(regExp).hasMatch(branch.name))) {
+ branches.add(branch.name);
+ }
+ }
+ return Body.forJson(<String, List<String>>{'Branches': branches});
+ }
+}
diff --git a/app_dart/lib/src/request_handlers/refresh_github_commits.dart b/app_dart/lib/src/request_handlers/refresh_github_commits.dart
index 8119d26..8dd2a22 100644
--- a/app_dart/lib/src/request_handlers/refresh_github_commits.dart
+++ b/app_dart/lib/src/request_handlers/refresh_github_commits.dart
@@ -18,6 +18,7 @@
import '../model/appengine/commit.dart';
import '../model/appengine/task.dart';
import '../model/devicelab/manifest.dart';
+import '../request_handlers/utils.dart';
import '../request_handling/api_request_handler.dart';
import '../request_handling/authentication.dart';
import '../request_handling/body.dart';
@@ -25,20 +26,6 @@
import '../service/datastore.dart';
import '../service/github_service.dart';
-/// Signature for a function that calculates the backoff duration to wait in
-/// between requests when GitHub responds with an error.
-///
-/// The `attempt` argument is zero-based, so if the first attempt to request
-/// from GitHub fails, and we're backing off before making the second attempt,
-/// the `attempt` argument will be zero.
-typedef GitHubBackoffCalculator = Duration Function(int attempt);
-
-/// Default backoff calculator.
-@visibleForTesting
-Duration twoSecondLinearBackoff(int attempt) {
- return const Duration(seconds: 2) * (attempt + 1);
-}
-
/// Queries GitHub for the list of recent commits according to different branches,
/// and creates corresponding rows in the cloud datastore and the BigQuery for any commits
/// not yet there. Then creates new task rows in the datastore for any commits that
@@ -72,7 +59,8 @@
const RepositorySlug slug = RepositorySlug('flutter', 'flutter');
final Stream<Branch> branches = github.repositories.listBranches(slug);
final DatastoreService datastore = datastoreProvider();
- final List<String> regExps = await _loadBranchRegExps();
+ final List<String> regExps = await loadBranchRegExps(
+ branchHttpClientProvider, log, gitHubBackoffCalculator);
await for (Branch branch in branches) {
if (regExps
@@ -275,42 +263,4 @@
throw HttpStatusException(
HttpStatus.serviceUnavailable, 'GitHub not responding');
}
-
- Future<List<String>> _loadBranchRegExps() async {
- const String path =
- '/flutter/cocoon/master/app_dart/dev/branch_regexps.txt';
- final Uri url = Uri.https('raw.githubusercontent.com', path);
-
- final HttpClient client = branchHttpClientProvider();
- try {
- for (int attempt = 0; attempt < 3; attempt++) {
- final HttpClientRequest clientRequest = await client.getUrl(url);
-
- try {
- final HttpClientResponse clientResponse = await clientRequest.close();
- final int status = clientResponse.statusCode;
-
- if (status == HttpStatus.ok) {
- final String content =
- await utf8.decoder.bind(clientResponse).join();
- return content
- .split('\n')
- .map((String branch) => branch.trim())
- .toList();
- } else {
- log.warning(
- 'Attempt to download branch_regexps.txt failed (HTTP $status)');
- return <String>['master'];
- }
- } catch (error, stackTrace) {
- log.error(
- 'Attempt to download branch_regexps.txt failed:\n$error\n$stackTrace');
- }
- await Future<void>.delayed(gitHubBackoffCalculator(attempt));
- }
- } finally {
- client.close(force: true);
- }
- return <String>['master'];
- }
}
diff --git a/app_dart/lib/src/request_handlers/utils.dart b/app_dart/lib/src/request_handlers/utils.dart
new file mode 100644
index 0000000..0ff39f6
--- /dev/null
+++ b/app_dart/lib/src/request_handlers/utils.dart
@@ -0,0 +1,65 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:appengine/appengine.dart';
+
+import '../foundation/typedefs.dart';
+
+/// Signature for a function that calculates the backoff duration to wait in
+/// between requests when GitHub responds with an error.
+///
+/// The [attempt] argument is zero-based, so if the first attempt to request
+/// from GitHub fails, and we're backing off before making the second attempt,
+/// the [attempt] argument will be zero.
+typedef GitHubBackoffCalculator = Duration Function(int attempt);
+
+/// Default backoff calculator.
+Duration twoSecondLinearBackoff(int attempt) {
+ return const Duration(seconds: 2) * (attempt + 1);
+}
+
+Future<List<String>> loadBranchRegExps(
+ HttpClientProvider branchHttpClientProvider,
+ Logging log,
+ GitHubBackoffCalculator gitHubBackoffCalculator) async {
+ const String path = '/flutter/cocoon/master/app_dart/dev/branch_regexps.txt';
+ final Uri url = Uri.https('raw.githubusercontent.com', path);
+
+ final HttpClient client = branchHttpClientProvider();
+ try {
+ for (int attempt = 0; attempt < 3; attempt++) {
+ final HttpClientRequest clientRequest = await client.getUrl(url);
+
+ try {
+ final HttpClientResponse clientResponse = await clientRequest.close();
+ final int status = clientResponse.statusCode;
+
+ if (status == HttpStatus.ok) {
+ final String content = await utf8.decoder.bind(clientResponse).join();
+ final List<String> branches = content
+ .split('\n')
+ .map((String branch) => branch.trim())
+ .toList();
+ branches.removeWhere((String branch) => branch.isEmpty);
+ return branches;
+ } else {
+ log.warning(
+ 'Attempt to download branch_regexps.txt failed (HTTP $status)');
+ }
+ } catch (error, stackTrace) {
+ log.error(
+ 'Attempt to download branch_regexps.txt failed:\n$error\n$stackTrace');
+ }
+ await Future<void>.delayed(gitHubBackoffCalculator(attempt));
+ }
+ } finally {
+ client.close(force: true);
+ }
+ log.error('GitHub not responding; giving up');
+ return <String>['master'];
+}
diff --git a/app_dart/test/request_handlers/get_branches_test.dart b/app_dart/test/request_handlers/get_branches_test.dart
new file mode 100644
index 0000000..e9e5cb1
--- /dev/null
+++ b/app_dart/test/request_handlers/get_branches_test.dart
@@ -0,0 +1,79 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:cocoon_service/src/request_handlers/get_branches.dart';
+import 'package:cocoon_service/src/request_handling/body.dart';
+import 'package:github/server.dart';
+import 'package:mockito/mockito.dart';
+import 'package:test/test.dart';
+
+import '../src/datastore/fake_cocoon_config.dart';
+import '../src/request_handling/fake_http.dart';
+import '../src/request_handling/request_handler_tester.dart';
+
+const String branchRegExp = '''
+ master
+ ^flutter-[0-9]+\.[0-9]+-candidate\.[0-9]+
+ ''';
+
+void main() {
+ group('GetBranches', () {
+ FakeConfig config;
+ FakeHttpClient branchHttpClient;
+ RequestHandlerTester tester;
+ GetBranches handler;
+ List<String> githubBranches;
+
+ Stream<Branch> branchStream() async* {
+ for (String branchName in githubBranches) {
+ final CommitDataUser author = CommitDataUser('a', 1, 'b');
+ final GitCommit gitCommit = GitCommit();
+ final CommitData commitData = CommitData('sha', gitCommit, 'test',
+ 'test', 'test', author, author, <Map<String, dynamic>>[]);
+ final Branch branch = Branch(branchName, commitData);
+ yield branch;
+ }
+ }
+
+ setUp(() {
+ final MockGitHub github = MockGitHub();
+ final MockRepositoriesService repositories = MockRepositoriesService();
+
+ const RepositorySlug slug = RepositorySlug('flutter', 'flutter');
+ config = FakeConfig(githubClient: github, flutterSlugValue: slug);
+ branchHttpClient = FakeHttpClient();
+ tester = RequestHandlerTester();
+ handler = GetBranches(
+ config,
+ branchHttpClientProvider: () => branchHttpClient,
+ gitHubBackoffCalculator: (int attempt) => Duration.zero,
+ );
+
+ when(github.repositories).thenReturn(repositories);
+ when(repositories.listBranches(slug)).thenAnswer((Invocation _) {
+ return branchStream();
+ });
+ });
+
+ test('returns branches matching regExps', () async {
+ githubBranches = <String>['flutter-1.1-candidate.1', 'master', 'test'];
+
+ branchHttpClient.request.response.body = branchRegExp;
+
+ final Body body = await tester.get(handler);
+ final Map<String, dynamic> result = await utf8.decoder
+ .bind(body.serialize())
+ .transform(json.decoder)
+ .single as Map<String, dynamic>;
+
+ expect(result['Branches'], <String>['flutter-1.1-candidate.1', 'master']);
+ });
+ });
+}
+
+class MockGitHub extends Mock implements GitHub {}
+
+class MockRepositoriesService extends Mock implements RepositoriesService {}
diff --git a/app_dart/test/request_handlers/refresh_github_commits_test.dart b/app_dart/test/request_handlers/refresh_github_commits_test.dart
index 8ddc1aa..d12212a 100644
--- a/app_dart/test/request_handlers/refresh_github_commits_test.dart
+++ b/app_dart/test/request_handlers/refresh_github_commits_test.dart
@@ -238,14 +238,6 @@
expect(tester.log.records.where(hasLevel(LogLevel.ERROR)), isNotEmpty);
});
});
- group('GitHubBackoffCalculator', () {
- test('twoSecondLinearBackoff', () {
- expect(twoSecondLinearBackoff(0), const Duration(seconds: 2));
- expect(twoSecondLinearBackoff(1), const Duration(seconds: 4));
- expect(twoSecondLinearBackoff(2), const Duration(seconds: 6));
- expect(twoSecondLinearBackoff(3), const Duration(seconds: 8));
- });
- });
}
String toSha(Commit commit) => commit.sha;
diff --git a/app_dart/test/request_handlers/utils_test.dart b/app_dart/test/request_handlers/utils_test.dart
new file mode 100644
index 0000000..541a33c
--- /dev/null
+++ b/app_dart/test/request_handlers/utils_test.dart
@@ -0,0 +1,79 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:appengine/appengine.dart';
+import 'package:test/test.dart';
+
+import 'package:cocoon_service/src/request_handlers/utils.dart';
+
+import '../src/request_handling/fake_http.dart';
+import '../src/request_handling/fake_logging.dart';
+
+const String branchRegExp = '''
+ master
+ ^flutter-[0-9]+\.[0-9]+-candidate\.[0-9]+
+ ''';
+
+void main() {
+ group('GetBranches', () {
+ FakeHttpClient branchHttpClient;
+ FakeLogging log;
+
+ setUp(() {
+ branchHttpClient = FakeHttpClient();
+ log = FakeLogging();
+ });
+
+ test('returns branches matching regExps', () async {
+ branchHttpClient.request.response.body = branchRegExp;
+ final List<String> branches = await loadBranchRegExps(
+ () => branchHttpClient, log, (int attempt) => Duration.zero);
+ expect(branches.length, 2);
+ });
+
+ test('retries regExps download upon HTTP failure', () async {
+ int retry = 0;
+ branchHttpClient.onIssueRequest = (FakeHttpClientRequest request) {
+ request.response.statusCode =
+ retry == 0 ? HttpStatus.serviceUnavailable : HttpStatus.ok;
+ retry++;
+ };
+
+ branchHttpClient.request.response.body = branchRegExp;
+ final List<String> branches = await loadBranchRegExps(
+ () => branchHttpClient, log, (int attempt) => Duration.zero);
+ expect(retry, 2);
+ expect(branches,
+ <String>['master', '^flutter-[0-9]+.[0-9]+-candidate.[0-9]+']);
+ expect(log.records.where(hasLevel(LogLevel.WARNING)), isNotEmpty);
+ expect(log.records.where(hasLevel(LogLevel.ERROR)), isEmpty);
+ });
+
+ test('gives up regExps download after 3 tries', () async {
+ int retry = 0;
+ branchHttpClient.onIssueRequest =
+ (FakeHttpClientRequest request) => retry++;
+ branchHttpClient.request.response.statusCode =
+ HttpStatus.serviceUnavailable;
+ branchHttpClient.request.response.body = branchRegExp;
+ final List<String> branches = await loadBranchRegExps(
+ () => branchHttpClient, log, (int attempt) => Duration.zero);
+ expect(branches, <String>['master']);
+ expect(retry, 3);
+ expect(log.records.where(hasLevel(LogLevel.WARNING)), isNotEmpty);
+ expect(log.records.where(hasLevel(LogLevel.ERROR)), isNotEmpty);
+ });
+ });
+
+ group('GitHubBackoffCalculator', () {
+ test('twoSecondLinearBackoff', () {
+ expect(twoSecondLinearBackoff(0), const Duration(seconds: 2));
+ expect(twoSecondLinearBackoff(1), const Duration(seconds: 4));
+ expect(twoSecondLinearBackoff(2), const Duration(seconds: 6));
+ expect(twoSecondLinearBackoff(3), const Duration(seconds: 8));
+ });
+ });
+}
diff --git a/app_dart/test/src/datastore/fake_cocoon_config.dart b/app_dart/test/src/datastore/fake_cocoon_config.dart
index daee7e7..f1a0b30 100644
--- a/app_dart/test/src/datastore/fake_cocoon_config.dart
+++ b/app_dart/test/src/datastore/fake_cocoon_config.dart
@@ -42,6 +42,7 @@
this.taskLogServiceAccountValue,
this.rollerAccountsValue,
this.luciTryInfraFailureRetriesValue,
+ this.flutterSlugValue,
FakeDatastoreDB dbValue,
}) : dbValue = dbValue ?? FakeDatastoreDB();
@@ -70,6 +71,7 @@
ServiceAccountCredentials taskLogServiceAccountValue;
Set<String> rollerAccountsValue;
int luciTryInfraFailureRetriesValue;
+ RepositorySlug flutterSlugValue;
@override
int get luciTryInfraFailureRetries => luciTryInfraFailureRetriesValue;
@@ -150,6 +152,9 @@
waitingForTreeToGoGreenLabelNameValue;
@override
+ RepositorySlug get flutterSlug => flutterSlugValue;
+
+ @override
Future<ServiceAccountCredentials> get taskLogServiceAccount async =>
taskLogServiceAccountValue;
