| // Copyright 2023 The Flutter Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:io'; |
| |
| import 'package:github/github.dart'; |
| import 'package:http/http.dart'; |
| |
| import 'debug_http.dart'; |
| import 'issue.dart'; |
| import 'team.dart'; |
| import 'utils.dart'; |
| |
| const bool _debugNetwork = false; |
| |
| // File that contains OAuth token obtained via https://github.com/settings/personal-access-tokens/new |
| // + Resource owner "flutter" |
| // + Public Repositories (read-only) |
| // + Organization permissions -> Members -> Read-only |
| final File tokenFile = File('.github-token'); |
| final File membersFile = File('members.txt'); |
| final File exmembersFile = File('exmembers.txt'); |
| |
| final Directory cache = Directory('cache'); |
| final Directory output = Directory('output'); |
| const String orgName = 'flutter'; |
| final RepositorySlug issueDatabaseRepo = RepositorySlug(orgName, 'flutter'); |
| final Set<RepositorySlug> repos = <RepositorySlug>{ |
| issueDatabaseRepo, // flutter/flutter |
| RepositorySlug(orgName, 'assets-for-api-docs'), |
| RepositorySlug(orgName, 'cocoon'), |
| RepositorySlug(orgName, 'codelabs'), |
| RepositorySlug(orgName, 'devtools'), |
| RepositorySlug(orgName, 'flutter-intellij'), |
| RepositorySlug(orgName, 'gallery'), |
| RepositorySlug(orgName, 'games'), |
| RepositorySlug(orgName, 'holobooth'), |
| RepositorySlug(orgName, 'io_flip'), |
| RepositorySlug(orgName, 'news_toolkit'), |
| RepositorySlug(orgName, 'packages'), |
| RepositorySlug(orgName, 'photobooth'), |
| RepositorySlug(orgName, 'pinball'), |
| RepositorySlug(orgName, 'platform_tests'), |
| RepositorySlug(orgName, 'samples'), |
| RepositorySlug(orgName, 'tests'), |
| RepositorySlug(orgName, 'website'), |
| }; |
| const String primaryTeam = 'flutter-hackers'; |
| const Duration rosterMaxAge = Duration(minutes: 10); |
| const Duration issueMaxAge = Duration(days: 365); |
| |
| const Set<String> csvSpecials = <String>{'\'', '"', ',', '\n'}; |
| |
| // Temporary workaround for https://github.com/SpinlockLabs/github.dart/issues/401 |
| bool issueIsOpen(final FullIssue issue) { |
| return issue.metadata.isOpen || issue.metadata.state.toUpperCase() == 'OPEN'; |
| } |
| |
| // Temporary workaround for https://github.com/SpinlockLabs/github.dart/issues/401 |
| bool issueIsClosed(final FullIssue issue) { |
| return issue.metadata.isClosed || |
| issue.metadata.state.toUpperCase() == 'CLOSED'; |
| } |
| |
| // Turns a username into an internal canonicalized form. |
| // We add a "👤" emoji here so that if we accidentally use the canonicalized form |
| // in the output, we will catch it. |
| String canon(final String? s) => '👤${(s ?? "").toLowerCase()}'; |
| |
| Future<int> full(final Directory cache, final GitHub github) async { |
| try { |
| // FETCH USER AND TEAM DATA |
| print('Team roster...'); |
| final roster = await TeamRoster.load( |
| cache: cache, |
| github: github, |
| orgName: orgName, |
| cacheEpoch: maxAge(rosterMaxAge), |
| ); |
| final allMembers = <String>{}; |
| final currentMembers = roster.teams[primaryTeam]!.keys.map(canon).toSet(); |
| final expectedMembers = (await membersFile.readAsString()) |
| .trimRight() |
| .split('\n') |
| .where((final String name) => !name.endsWith(' (DO NOT ADD)')) |
| .map(canon) |
| .toSet(); |
| final expectedExmembers = (await exmembersFile.readAsString()) |
| .trimRight() |
| .split('\n') |
| .map(canon) |
| .toSet(); |
| try { |
| final unexpectedMembers = currentMembers.difference(expectedMembers); |
| final memberExmembers = expectedExmembers.intersection(currentMembers); |
| final missingMembers = expectedMembers.difference(currentMembers); |
| if (unexpectedMembers.isNotEmpty) { |
| print( |
| 'WARNING: The following users are currently members of $primaryTeam but not expected: ${unexpectedMembers.join(', ')}', |
| ); |
| } |
| if (memberExmembers.isNotEmpty) { |
| print( |
| 'WARNING: The following users are currently members of $primaryTeam but should have been removed: ${memberExmembers.join(', ')}', |
| ); |
| } |
| if (missingMembers.isNotEmpty) { |
| print( |
| 'WARNING: The following users are currently NOT members of $primaryTeam but were expected:\n ${missingMembers.join('\n ')}', |
| ); |
| } |
| allMembers |
| ..addAll(currentMembers) |
| ..addAll(expectedMembers) |
| ..addAll(expectedExmembers); |
| } on FileSystemException catch (e) { |
| if (membersFile.existsSync()) { |
| print('Unable to read ${membersFile.path}: ${e.message}'); |
| return 1; |
| } |
| } |
| for (final teamName in roster.teams.keys.where( |
| (final String? team) => team != null, |
| )) { |
| for (final userName in roster.teams[teamName]!.keys) { |
| if (!roster.teams[null]!.containsKey(userName)) { |
| print( |
| 'WARNING: user $userName is in $teamName but not in organization.', |
| ); |
| } |
| } |
| } |
| |
| // FETCH ACTIVITY |
| print(''); |
| print('Fetching issues...'); |
| final issues = <String, Map<int, FullIssue>>{}; |
| try { |
| for (final repo in repos) { |
| await fetchAllIssues( |
| github, |
| cache, |
| repo, |
| issueMaxAge, |
| issues[repo.fullName] = <int, FullIssue>{}, |
| ); |
| } |
| print('Updating issues...'); |
| for (final repo in repos) { |
| await updateAllIssues(github, cache, repo, issues[repo.fullName]!); |
| } |
| } on Abort {} // ignore: empty_catches |
| |
| // ANALYZE ACTIVITY RESULTS |
| print(''); |
| print('Analyzing...'); |
| try { |
| await output.create(recursive: true); |
| } on FileSystemException catch (e) { |
| print('Unable to create output in "${output.path}": $e'); |
| return 1; |
| } |
| final activityMetrics = <String, UserActivity>{}; |
| UserActivity forUser(final User? user) { |
| return activityMetrics.putIfAbsent(user!.login!, () { |
| final result = UserActivity(); |
| if (expectedMembers.contains(canon(user.login))) { |
| result |
| ..isMember = true |
| ..isActiveMember = true; |
| } else if (expectedExmembers.contains(canon(user.login))) { |
| result.isMember = true; |
| } |
| return result; |
| }); |
| } |
| |
| final reactionKinds = <String>{}; |
| final foundPriorities = <String?>{}; |
| void increment<T>(final Map<T, int> map, final Set<T> keys, final T key) { |
| keys.add(key); |
| if (map.containsKey(key)) { |
| map[key] = map[key]! + 1; |
| } else { |
| map[key] = 1; |
| } |
| } |
| |
| roster.teams[primaryTeam]!.values.forEach(forUser); |
| |
| final allIssues = issues.values |
| .expand((final Map<int, FullIssue> issues) => issues.values) |
| .where((final FullIssue issue) => issue.isValid) |
| .toList(); |
| for (final issue in allIssues) { |
| if (issue.isPullRequest) { |
| // Pull requests filed. |
| forUser(issue.metadata.user).pullRequests.add(issue.metadata.createdAt); |
| } else { |
| // Issues filed by users. |
| forUser(issue.metadata.user).issues.add(issue.metadata.createdAt); |
| increment<String?>( |
| forUser(issue.metadata.user).priorityCount, |
| foundPriorities, |
| issue.priority, |
| ); |
| } |
| |
| // Pull request comments. |
| for (final comment in issue.comments) { |
| // Comments left by users on pull requests. |
| // Issue comments have a lot of spam, excluding for now. |
| if (issue.isPullRequest) { |
| forUser(comment.user).comments.add(comment.createdAt); |
| } |
| } |
| } |
| DateTime? earliest; |
| DateTime? latest; |
| void considerTimes( |
| final UserActivity activity, |
| final List<DateTime?> times, |
| ) { |
| for (final time in times) { |
| if (activity.earliest == null || |
| (time != null && time.isBefore(activity.earliest!))) { |
| activity.earliest = time; |
| } |
| if (activity.latest == null || |
| (time != null && time.isAfter(activity.latest!))) { |
| activity.latest = time; |
| } |
| if (earliest == null || (time != null && time.isBefore(earliest!))) { |
| earliest = time; |
| } |
| if (latest == null || (time != null && time.isAfter(latest!))) { |
| latest = time; |
| } |
| } |
| } |
| |
| for (final activity in activityMetrics.values) { |
| considerTimes(activity, activity.issues); |
| considerTimes(activity, activity.comments); |
| considerTimes(activity, activity.pullRequests); |
| } |
| |
| // PRINT ACTIVITY RESULTS |
| final summary = StringBuffer(); |
| for (final reactionKind in reactionKinds) { |
| verifyStringSanity(reactionKind, csvSpecials); |
| } |
| final sortedReactionKinds = reactionKinds.toList()..sort(); |
| summary.writeln( |
| 'user,is member,is active member,earliest,latest,days active,total,density,issues,comments,closures,self closures,pull requests,characters,missing priority,${priorities.join(',')},reactions,${sortedReactionKinds.join(',')}', |
| ); |
| var usersWithMoreThanOneDayActive = 0; |
| for (final user |
| in activityMetrics.keys.toList()..sort( |
| (final String a, final String b) => |
| activityMetrics[b]!.total - activityMetrics[a]!.total, |
| )) { |
| verifyStringSanity(user, csvSpecials); |
| final activity = activityMetrics[user]!; |
| if (activity.daysActive > 0) { |
| usersWithMoreThanOneDayActive += 1; |
| } |
| summary.write( |
| '$user,${activity.isMember},${activity.isActiveMember},${activity.earliest},${activity.latest},${activity.daysActive},${activity.total},${activity.density},${activity.issues.length},${activity.comments.length},${activity.closures.length},${activity.selfClosures},${activity.pullRequests.length},${activity.characters},${activity.priorityCount[null] ?? 0}', |
| ); |
| for (final priority in priorities) { |
| summary.write(',${activity.priorityCount[priority] ?? 0}'); |
| } |
| summary.write(',${activity.reactions.length}'); |
| for (final reactionKind in sortedReactionKinds) { |
| summary.write(',${activity.reactionCount[reactionKind] ?? 0}'); |
| } |
| summary.writeln(); |
| } |
| await File('${output.path}/users.csv').writeAsString(summary.toString()); |
| print('Total participants: ${activityMetrics.length}'); |
| print( |
| 'Participants with more than one day of activity: $usersWithMoreThanOneDayActive', |
| ); |
| print('User activity results stored in: ${output.path}/users.csv'); |
| |
| // ANALYZE PRIORITIES |
| final priorityAnalysis = <String, PriorityResults>{}; |
| for (final priority in priorities) { |
| priorityAnalysis[priority] = PriorityResults(); |
| } |
| |
| final primaryIssues = issues[issueDatabaseRepo.fullName]!.values |
| .where((final issue) => issue.isValid && !issue.isPullRequest) |
| .toList(); |
| final primaryPRs = issues[issueDatabaseRepo.fullName]!.values |
| .where((final issue) => issue.isValid && issue.isPullRequest) |
| .toList(); |
| for (final issue in primaryIssues.where( |
| (final issue) => issue.priority != null, |
| )) { |
| final priorityResults = priorityAnalysis[issue.priority!]!; |
| final teamIssue = allMembers.contains(canon(issue.metadata.user!.login)); |
| priorityResults.total += 1; |
| if (teamIssue) { |
| priorityResults.openedByTeam += 1; |
| } else { |
| priorityResults.openedByNonTeam += 1; |
| } |
| if (issueIsOpen(issue)) { |
| priorityResults.open += 1; |
| } else { |
| priorityResults.closed += 1; |
| if (issue.metadata.closedAt == null || |
| issue.metadata.createdAt == null) { |
| print( |
| 'WARNING: bogus open/close timeline data in ${issue.issueNumber}: opened at ${issue.metadata.createdAt}, closed at ${issue.metadata.closedAt}, state: ${issue.metadata.state}', |
| ); |
| } else { |
| final timeOpen = issue.metadata.closedAt!.difference( |
| issue.metadata.createdAt!, |
| ); |
| priorityResults.timeOpen.add(timeOpen); |
| } |
| if (teamIssue) { |
| priorityResults.openedByTeamAndClosed += 1; |
| } else { |
| priorityResults.openedByNonTeamAndClosed += 1; |
| } |
| } |
| } |
| |
| // PRINT PRIORITY RESULTS |
| summary |
| ..clear() |
| ..writeln( |
| 'priority,total,open,closed,openedByTeam,openedByNonTeam,openedByTeamAndClosed,openedByNonTeamAndClosed,meanTimeOpen,p01TimeOpen,p05TimeOpen,medianTimeOpen,p95TimeOpen,p99TimeOpen', |
| ); |
| for (final priority in priorities) { |
| verifyStringSanity(priority, csvSpecials); |
| final entry = priorityAnalysis[priority]!; |
| summary.write( |
| '$priority,${entry.total},${entry.open},${entry.closed},${entry.openedByTeam},${entry.openedByNonTeam},${entry.openedByTeamAndClosed},${entry.openedByNonTeamAndClosed},', |
| ); |
| if (entry.timeOpen.isEmpty) { |
| summary.write('NaN,NaN,NaN,NaN'); |
| } else { |
| entry.timeOpen.sort(); |
| summary |
| ..write( |
| '${entry.timeOpen.fold<int>(0, (final int sum, final Duration t) => sum + t.inMilliseconds) / (entry.timeOpen.length * Duration.millisecondsPerDay)},', |
| ) |
| ..write( |
| '${entry.timeOpen[(entry.timeOpen.length * 0.01).floor()].inMilliseconds / Duration.millisecondsPerDay},', |
| ) |
| ..write( |
| '${entry.timeOpen[(entry.timeOpen.length * 0.05).floor()].inMilliseconds / Duration.millisecondsPerDay},', |
| ); |
| if (entry.timeOpen.length > 1) { |
| final median1 = entry.timeOpen[(entry.timeOpen.length / 2.0).floor()]; |
| final median2 = entry.timeOpen[(entry.timeOpen.length / 2.0).ceil()]; |
| summary.write( |
| '${(median1.inMilliseconds + median2.inMilliseconds) / (2.0 * Duration.millisecondsPerDay)},', |
| ); |
| } else { |
| summary.write( |
| '${(entry.timeOpen.first.inMilliseconds) / Duration.millisecondsPerDay},', |
| ); |
| } |
| summary |
| ..write( |
| '${entry.timeOpen[(entry.timeOpen.length * 0.95).floor()].inMilliseconds / Duration.millisecondsPerDay},', |
| ) |
| ..write( |
| '${entry.timeOpen[(entry.timeOpen.length * 0.99).floor()].inMilliseconds / Duration.millisecondsPerDay},', |
| ); |
| } |
| summary.writeln(); |
| } |
| await File( |
| '${output.path}/priorities.csv', |
| ).writeAsString(summary.toString()); |
| print('Priority results stored in: ${output.path}/priorities.csv'); |
| |
| // PRINT ISSUE DATA |
| var deadCount = 0; |
| var zombieCount = 0; |
| summary |
| ..clear() |
| ..writeln( |
| 'repository,issue,state,createdAt,createdBy,closedAt,closedBy,timeOpen,updatedAt,priority,labelCount,commentCount,${sortedReactionKinds.join(',')},daysToTwentyVotes,isNewFeature,isProposal,isPendingAutoclosure,isFiledByTeam,isFiledByExMember,', |
| ); |
| for (final issue in allIssues.where( |
| (final FullIssue issue) => !issue.isPullRequest, |
| )) { |
| verifyStringSanity(issue.metadata.state, csvSpecials); |
| summary.write( |
| '${issue.repo.fullName},${issue.issueNumber},${issue.metadata.state},${issue.metadata.createdAt},${issue.metadata.user!.login},${issue.metadata.closedAt},${issue.metadata.closedBy?.login ?? (issue.isValid && issueIsClosed(issue) ? "<unknown>" : "")},${issue.isValid && issueIsClosed(issue) ? issue.metadata.closedAt!.difference(issue.metadata.createdAt!).inMilliseconds / Duration.millisecondsPerDay : ""},${issue.metadata.updatedAt},${issue.priority ?? ""},${issue.labels.length},${issue.comments.length}', |
| ); |
| for (final reactionKind in sortedReactionKinds) { |
| var count = 0; |
| for (final reaction in issue.reactions) { |
| if (reaction.content == reactionKind) { |
| count += 1; |
| } |
| } |
| summary.write(',$count'); |
| } |
| var count = 0; |
| int? daysToTwentyVotes; |
| for (final reaction in issue.reactions) { |
| if (reaction.content == '+1') { |
| count += 1; |
| if (count >= 20) { |
| daysToTwentyVotes = reaction.createdAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays; |
| break; |
| } |
| } |
| } |
| summary |
| ..write(',${daysToTwentyVotes ?? ''}') |
| ..write(',${issue.labels.contains('new feature')}') |
| ..write(',${issue.labels.contains('proposal')}') |
| ..write(',${issue.labels.contains('waiting for customer response')}') |
| ..write(',${allMembers.contains(canon(issue.metadata.user!.login))}') |
| ..write( |
| ',${expectedExmembers.contains(canon(issue.metadata.user!.login))}', |
| ) |
| ..writeln(); |
| if ((daysToTwentyVotes == null || daysToTwentyVotes > 60) && |
| issue.labels.contains('new feature') && |
| (issueIsOpen(issue) || |
| issue.metadata.closedAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays > |
| 60)) { |
| if (issueIsOpen(issue)) { |
| deadCount += 1; |
| } else { |
| zombieCount += 1; |
| } |
| } |
| } |
| await File('${output.path}/issues.csv').writeAsString(summary.toString()); |
| print('Issue summaries stored in: ${output.path}/issues.csv'); |
| print( |
| '$deadCount issues would be closed; $zombieCount issues would not have been fixed.', |
| ); |
| |
| // COLLECT CLOSE TIME PERCENTILES |
| var maxDaysToClose = 0; |
| final closureTimeHistogramClosed = <int, Map<String?, int>>{}; |
| final closureTimeTotalsClosed = <String?, int>{ |
| null: 0, |
| for (final String priority in priorities) priority: 0, |
| }; |
| for (final issue in primaryIssues.where(issueIsClosed)) { |
| final timeOpen = issue.metadata.closedAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays; |
| final priority = issue.priority; |
| closureTimeHistogramClosed |
| .putIfAbsent(timeOpen, () => <String?, int>{}) |
| .update(priority, (final int value) => value + 1, ifAbsent: () => 1); |
| closureTimeTotalsClosed[priority] = |
| closureTimeTotalsClosed[priority]! + 1; |
| if (timeOpen > maxDaysToClose) { |
| maxDaysToClose = timeOpen; |
| } |
| } |
| |
| // PRINT CLOSE TIME PERCENTILES OF CLOSED BUGS |
| summary |
| ..clear() |
| ..writeln( |
| 'time to close (days),unprioritized,${priorities.where((final String priority) => closureTimeTotalsClosed[priority]! > 0).join(",")},unprioritized,${priorities.where((final String priority) => closureTimeTotalsClosed[priority]! > 0).join(",")}', |
| ); |
| if (closureTimeTotalsClosed[null]! > 0) { |
| final closureTimeCumulativeSum = <String?, int>{ |
| null: 0, |
| for (final String priority in priorities) |
| if (closureTimeTotalsClosed[priority]! > 0) priority: 0, |
| }; |
| for (var day = 0; day <= maxDaysToClose; day += 1) { |
| if (closureTimeHistogramClosed.containsKey(day)) { |
| if (closureTimeHistogramClosed[day]!.containsKey(null)) { |
| closureTimeCumulativeSum[null] = |
| closureTimeCumulativeSum[null]! + |
| closureTimeHistogramClosed[day]![null]!; |
| } |
| for (final priority in priorities) { |
| if (closureTimeHistogramClosed[day]!.containsKey(priority)) { |
| closureTimeCumulativeSum[priority] = |
| closureTimeCumulativeSum[priority]! + |
| closureTimeHistogramClosed[day]![priority]!; |
| } |
| } |
| } |
| summary.write('$day,${closureTimeCumulativeSum[null]}'); |
| for (final priority in priorities) { |
| if (closureTimeTotalsClosed[priority]! > 0) { |
| summary.write(',${closureTimeCumulativeSum[priority]}'); |
| } |
| } |
| summary.write( |
| ',${100.0 * closureTimeCumulativeSum[null]! / closureTimeTotalsClosed[null]!}%', |
| ); |
| for (final priority in priorities) { |
| if (closureTimeTotalsClosed[priority]! > 0) { |
| summary.write( |
| ',${100.0 * closureTimeCumulativeSum[priority]! / closureTimeTotalsClosed[priority]!}%', |
| ); |
| } |
| } |
| summary.writeln(); |
| } |
| } |
| await File( |
| '${output.path}/priority-percentiles.csv', |
| ).writeAsString(summary.toString()); |
| print( |
| 'Priority percentiles stored in: ${output.path}/priority-percentiles.csv', |
| ); |
| |
| // COLLECT CLOSE TIME PERCENTILES OF ALL BUGS |
| final closureTimeHistogramAll = <int, Map<String?, int>>{}; |
| final closureTimeTotalsAll = <String?, int>{ |
| null: 0, |
| for (final String priority in priorities) priority: 0, |
| }; |
| for (final issue in primaryIssues) { |
| final priority = issue.priority; |
| closureTimeTotalsAll[priority] = closureTimeTotalsAll[priority]! + 1; |
| final timeOpen = issueIsClosed(issue) |
| ? issue.metadata.closedAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays |
| : maxDaysToClose + 1; |
| closureTimeHistogramAll |
| .putIfAbsent(timeOpen, () => <String?, int>{}) |
| .update(priority, (final int value) => value + 1, ifAbsent: () => 1); |
| } |
| |
| // PRINT CLOSE TIME PERCENTILES OF ALL BUGS |
| summary |
| ..clear() |
| ..writeln( |
| 'time to close (days),unprioritized,${priorities.where((final String priority) => closureTimeTotalsAll[priority]! > 0).join(",")},unprioritized,${priorities.where((final String priority) => closureTimeTotalsAll[priority]! > 0).join(",")}', |
| ); |
| if (closureTimeTotalsAll[null]! > 0) { |
| final closureTimeCumulativeSum = <String?, int>{ |
| null: 0, |
| for (final String priority in priorities) |
| if (closureTimeTotalsAll[priority]! > 0) priority: 0, |
| }; |
| for (var day = 0; day <= maxDaysToClose + 1; day += 1) { |
| if (closureTimeHistogramAll.containsKey(day)) { |
| if (closureTimeHistogramAll[day]!.containsKey(null)) { |
| closureTimeCumulativeSum[null] = |
| closureTimeCumulativeSum[null]! + |
| closureTimeHistogramAll[day]![null]!; |
| } |
| for (final priority in priorities) { |
| if (closureTimeHistogramAll[day]!.containsKey(priority)) { |
| closureTimeCumulativeSum[priority] = |
| closureTimeCumulativeSum[priority]! + |
| closureTimeHistogramAll[day]![priority]!; |
| } |
| } |
| } |
| summary.write('$day,${closureTimeCumulativeSum[null]}'); |
| for (final priority in priorities) { |
| if (closureTimeTotalsAll[priority]! > 0) { |
| summary.write(',${closureTimeCumulativeSum[priority]}'); |
| } |
| } |
| summary.write( |
| ',${100.0 * closureTimeCumulativeSum[null]! / closureTimeTotalsAll[null]!}%', |
| ); |
| for (final priority in priorities) { |
| if (closureTimeTotalsAll[priority]! > 0) { |
| summary.write( |
| ',${100.0 * closureTimeCumulativeSum[priority]! / closureTimeTotalsAll[priority]!}%', |
| ); |
| } |
| } |
| summary.writeln(); |
| } |
| } |
| await File( |
| '${output.path}/priority-percentiles-all.csv', |
| ).writeAsString(summary.toString()); |
| print( |
| 'Priority percentiles stored in: ${output.path}/priority-percentiles-all.csv', |
| ); |
| |
| // PRINT PR DATA |
| summary |
| ..clear() |
| ..writeln( |
| 'repository,pr,user,state,createdAt,closedAt,timeOpen,updatedAt,labelCount,commentCount,${sortedReactionKinds.join(',')}', |
| ); |
| for (final issue in allIssues.where( |
| (final FullIssue issue) => issue.isPullRequest, |
| )) { |
| verifyStringSanity(issue.metadata.state, csvSpecials); |
| summary.write( |
| '${issue.repo.fullName},${issue.issueNumber},${issue.metadata.user!.login},${issue.metadata.state},${issue.metadata.createdAt},${issue.metadata.closedAt},${issueIsClosed(issue) ? issue.metadata.closedAt!.difference(issue.metadata.createdAt!).inMilliseconds / Duration.millisecondsPerDay : ""},${issue.metadata.updatedAt},${issue.labels.length},${issue.comments.length}', |
| ); |
| for (final reactionKind in sortedReactionKinds) { |
| var count = 0; |
| for (final reaction in issue.reactions) { |
| if (reaction.content == reactionKind) { |
| count += 1; |
| } |
| } |
| summary.write(',$count'); |
| } |
| summary.writeln(); |
| } |
| await File('${output.path}/prs.csv').writeAsString(summary.toString()); |
| print('PR summaries stored in: ${output.path}/prs.csv'); |
| |
| // PRINT USERS |
| final teamNames = |
| roster.teams.keys |
| .where((final String? name) => name != null) |
| .cast<String>() |
| .toList() |
| ..sort(); |
| final userNames = roster.teams[null]!.keys.toList()..sort(); |
| summary |
| ..clear() |
| ..writeln('user,${teamNames.join(',')}'); |
| for (final userName in userNames) { |
| verifyStringSanity(userName, csvSpecials); |
| summary.write(userName); |
| for (final String? teamName in teamNames) { |
| summary.write(','); |
| if (roster.teams[teamName]!.containsKey(userName)) { |
| summary.write('1'); |
| } else { |
| summary.write('0'); |
| } |
| } |
| summary.writeln(); |
| } |
| await File('${output.path}/teams.csv').writeAsString(summary.toString()); |
| print('Team membership summaries stored in: ${output.path}/teams.csv'); |
| |
| // WEEKLY ACTIVITY OVER TIME |
| if (earliest != null) { |
| assert(latest != null, 'invariant violation'); |
| const window = Duration.millisecondsPerDay * 7; |
| final firstWeekStart = earliest!.millisecondsSinceEpoch ~/ window; |
| final weeks = List<WeekActivity>.generate( |
| 1 + (latest!.millisecondsSinceEpoch ~/ window) - firstWeekStart, |
| (final int index) => WeekActivity( |
| DateTime.fromMillisecondsSinceEpoch( |
| (index + firstWeekStart) * window, |
| ), |
| reactionKinds, |
| priorities, |
| ), |
| ); |
| WeekActivity? forWeek(final DateTime? time) { |
| if (time == null) { |
| return null; |
| } |
| return weeks[(time.millisecondsSinceEpoch ~/ window) - firstWeekStart]; |
| } |
| |
| for (final issue in allIssues) { |
| if (!issue.isValid) { |
| continue; |
| } |
| if (issue.isPullRequest) { |
| forWeek(issue.metadata.createdAt)!.pullRequests += 1; |
| } else { |
| forWeek(issue.metadata.createdAt)!.issues += 1; |
| forWeek(issue.metadata.createdAt)!.priorityCount[issue.priority] = |
| forWeek( |
| issue.metadata.createdAt, |
| )!.priorityCount[issue.priority]! + |
| 1; |
| if (issueIsOpen(issue)) { |
| forWeek(issue.metadata.createdAt)!.remainingIssues += 1; |
| } |
| } |
| if (!issue.isPullRequest && (issue.metadata.closedBy != null)) { |
| forWeek(issue.metadata.closedAt)?.closures += 1; |
| if (issue.metadata.closedBy!.login == issue.metadata.user!.login) { |
| forWeek(issue.metadata.closedAt)?.selfClosures += 1; |
| } |
| } |
| forWeek(issue.metadata.createdAt)?.characters += |
| issue.metadata.body.length; |
| for (final comment in issue.comments) { |
| forWeek(comment.createdAt)!.comments += 1; |
| forWeek(comment.createdAt)!.characters += comment.body!.length; |
| } |
| } |
| if (weeks.isNotEmpty) { |
| weeks.removeLast(); // last week is incomplete data |
| } |
| |
| // PRINT WEEKLY ACTIVITY |
| summary |
| ..clear() |
| ..writeln( |
| 'week,total,issues,remaining issues,closures,self closures,net issues opened,comments,pull requests,characters,missing priority,${priorities.join(',')},reactions,${sortedReactionKinds.join(',')}', |
| ); |
| for (final week in weeks) { |
| verifyStringSanity(week.start.toIso8601String(), csvSpecials); |
| summary.write( |
| '${week.start},${week.total},${week.issues},${week.remainingIssues},${week.closures},${week.selfClosures},${week.issues - week.closures},${week.comments},${week.pullRequests},${week.characters},${week.priorityCount[null]!}', |
| ); |
| for (final priority in priorities) { |
| summary.write(',${week.priorityCount[priority]!}'); |
| } |
| summary.write(',${week.reactions}'); |
| for (final reactionKind in sortedReactionKinds) { |
| summary.write(',${week.reactionCount[reactionKind]!}'); |
| } |
| summary.writeln(); |
| } |
| await File('${output.path}/weeks.csv').writeAsString(summary.toString()); |
| print('Weekly activity results stored in: ${output.path}/weeks.csv'); |
| } |
| |
| // COLLECT LABELS DATA |
| final labels = <String, LabelData>{}; |
| final now = DateTime.now(); |
| for (final issue in primaryIssues) { |
| for (final label in issue.metadata.labels) { |
| final data = labels.putIfAbsent(label.name, () => LabelData(label.name)) |
| ..all += 1 |
| ..issues += 1; |
| if (issueIsOpen(issue)) { |
| data.open += 1; |
| } |
| if (issueIsClosed(issue)) { |
| data.closed += 1; |
| } |
| if (now.difference(issue.metadata.updatedAt!) < |
| const Duration(days: 52 * 7)) { |
| data.issuesUpdated52 += 1; |
| if (now.difference(issue.metadata.updatedAt!) < |
| const Duration(days: 12 * 7)) { |
| data.issuesUpdated12 += 1; |
| } |
| } |
| } |
| } |
| for (final issue in primaryPRs) { |
| for (final label in issue.metadata.labels) { |
| final data = labels.putIfAbsent(label.name, () => LabelData(label.name)) |
| ..all += 1 |
| ..prs += 1; |
| if (now.difference(issue.metadata.updatedAt!) < |
| const Duration(days: 52 * 7)) { |
| data.prsUpdated52 += 1; |
| if (now.difference(issue.metadata.updatedAt!) < |
| const Duration(days: 12 * 7)) { |
| data.prsUpdated12 += 1; |
| } |
| } |
| } |
| } |
| |
| // PRINT LABELS DATA |
| summary |
| ..clear() |
| ..writeln( |
| 'label,issues and PRs,all issues,open issues,closed issues,issues updated in last 12 weeks,issues updated in last 52 weeks,all PRs,PRs updated in last 12 weeks,PRs updated in last 52 weeks', |
| ); |
| for (final label in labels.values) { |
| verifyStringSanity(label.name, csvSpecials); |
| summary.writeln( |
| '${label.name},${label.all},${label.issues},${label.open},${label.closed},${label.issuesUpdated12},${label.issuesUpdated52},${label.prs},${label.prsUpdated12},${label.prsUpdated52}', |
| ); |
| } |
| await File('${output.path}/labels.csv').writeAsString(summary.toString()); |
| print('Labels stored in: ${output.path}/labels.csv'); |
| |
| return 0; |
| } on Abort { |
| print(''); |
| return 2; |
| // ignore: avoid_catches_without_on_clauses |
| } catch (e, stack) { |
| print('\nFatal error (${e.runtimeType}).'); |
| print('$e\n$stack'); |
| return 1; |
| } |
| } |
| |
| class LabelData { |
| LabelData(this.name); |
| final String name; |
| int all = 0; |
| int issues = 0; |
| int open = 0; |
| int closed = 0; |
| int issuesUpdated12 = 0; |
| int issuesUpdated52 = 0; |
| int prs = 0; |
| int prsUpdated12 = 0; |
| int prsUpdated52 = 0; |
| } |
| |
| class WeekActivity { |
| WeekActivity( |
| this.start, |
| final Set<String> reactionKinds, |
| final List<String> priorities, |
| ) { |
| priorityCount[null] = 0; |
| for (final priority in priorities) { |
| priorityCount[priority] = 0; |
| } |
| for (final reactionKind in reactionKinds) { |
| reactionCount[reactionKind] = 0; |
| } |
| } |
| final DateTime start; |
| int issues = 0; |
| int comments = 0; |
| int closures = 0; |
| int remainingIssues = 0; |
| int pullRequests = 0; |
| int reactions = 0; |
| Map<String, int> reactionCount = <String, int>{}; |
| Map<String?, int> priorityCount = <String?, int>{}; |
| int selfClosures = 0; |
| int characters = 0; |
| |
| int get total => issues + comments + closures + pullRequests + reactions; |
| } |
| |
| class UserActivity { |
| bool isMember = false; |
| bool isActiveMember = false; |
| List<DateTime?> issues = <DateTime?>[]; |
| List<DateTime?> comments = <DateTime?>[]; |
| List<DateTime?> closures = <DateTime?>[]; |
| List<DateTime?> pullRequests = <DateTime?>[]; |
| List<DateTime?> reactions = <DateTime?>[]; |
| Map<String, int> reactionCount = <String, int>{}; |
| Map<String?, int> priorityCount = <String?, int>{}; |
| int selfClosures = 0; |
| int characters = 0; |
| |
| DateTime? earliest; |
| DateTime? latest; |
| |
| int get total => |
| issues.length + |
| comments.length + |
| closures.length + |
| pullRequests.length + |
| reactions.length; |
| double get density => (earliest == null || latest == null) |
| ? double.nan |
| : total / |
| (latest!.millisecondsSinceEpoch - earliest!.millisecondsSinceEpoch); |
| double get daysActive => (earliest == null || latest == null) |
| ? double.nan |
| : (latest!.millisecondsSinceEpoch - earliest!.millisecondsSinceEpoch) / |
| Duration.millisecondsPerDay; |
| } |
| |
| class PriorityResults { |
| int total = 0; |
| int open = 0; |
| int closed = 0; |
| int openedByTeam = 0; |
| int openedByNonTeam = 0; |
| int openedByTeamAndClosed = 0; |
| int openedByNonTeamAndClosed = 0; |
| final List<Duration> timeOpen = <Duration>[]; |
| } |
| |
| Future<int> main(final List<String> arguments) async { |
| print(''); |
| print('GitHub Repository Analysis'); |
| print('=========================='); |
| print(''); |
| ProcessSignal.sigint.watch().listen((final ProcessSignal signal) { |
| stdout.write('\x1B[K\r'); |
| switch (mode) { |
| case Mode.full: |
| print('Skipping full update...'); |
| mode = Mode.abbreviated; |
| case Mode.abbreviated: |
| mode = Mode.aborted; |
| print('Skipping to generation...'); |
| aborter.complete(); |
| case Mode.aborted: |
| print('Terminating immediately!'); |
| exit(2); |
| } |
| }); |
| try { |
| await cache.create(recursive: true); |
| } on FileSystemException catch (e) { |
| print('Unable to create cache in "${cache.path}": $e'); |
| return 1; |
| } |
| final client = _debugNetwork ? DebugHttpClient() : Client(); |
| late final GitHub github; |
| try { |
| final token = await tokenFile.readAsString(); |
| github = GitHub(auth: Authentication.withToken(token), client: client); |
| } on FileSystemException catch (e) { |
| if (tokenFile.existsSync()) { |
| print('Unable to read ${tokenFile.path}: ${e.message}'); |
| return 1; |
| } |
| print('No token file; connecting to GitHub anonymously...'); |
| print(''); |
| github = GitHub(client: client); |
| } |
| if (arguments.isEmpty) { |
| return full(cache, github); |
| } |
| for (final argument in arguments) { |
| final parts = argument.split(':'); |
| if (parts.isNotEmpty && parts[0] == 'issue') { |
| if (parts.length != 4) { |
| print( |
| 'Not sure what to do with "$argument" (format for issue is issue:org:repo:number).', |
| ); |
| exit(1); |
| } |
| final repo = RepositorySlug(parts[1], parts[2]); |
| final issueNumber = int.tryParse(parts[3], radix: 10); |
| if (issueNumber == null) { |
| print( |
| 'Not sure what to do with "$argument" (fourth component is not a number).', |
| ); |
| exit(1); |
| } |
| final issue = await FullIssue.load( |
| cache: cache, |
| github: github, |
| repo: repo, |
| issueNumber: issueNumber, |
| cacheEpoch: DateTime.now().subtract(const Duration(hours: 24)), |
| ); |
| final summary = StringBuffer(); |
| final thumbs = List<int>.filled( |
| issue.reactions.last.createdAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays + |
| 1, |
| 0, |
| ); |
| for (final reaction in issue.reactions) { |
| if (reaction.content == '+1') { |
| final day = reaction.createdAt! |
| .difference(issue.metadata.createdAt!) |
| .inDays; |
| thumbs[day] = thumbs[day] + 1; |
| } |
| } |
| summary.writeln('day,thumbs,sum'); |
| var sum = 0; |
| for (var day = 0; day < thumbs.length; day += 1) { |
| sum += thumbs[day]; |
| summary.writeln('$day,${thumbs[day]},$sum'); |
| } |
| await File( |
| '${output.path}/issue:${repo.owner}:${repo.name}:$issueNumber:thumbs:history.csv', |
| ).writeAsString(summary.toString()); |
| } |
| } |
| return 0; |
| } |
