blob: e6f57a05acc25a2bf6a804c5de7ea2dad3c30b1f [file] [log] [blame] [edit]
// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:discoveryapis_generator/discoveryapis_generator.dart';
// ignore: implementation_imports
import 'package:discoveryapis_generator/src/namer.dart';
import 'package:yaml/yaml.dart';
import 'package:yaml_edit/yaml_edit.dart';
import '../googleapis_generator.dart';
import 'fetch_core.dart';
class Package {
final String name;
final List<String> apis;
final Pubspec pubspec;
final String readme;
final String license;
final String changelog;
final String? example;
final String? monoPkg;
Package(
this.name,
this.apis,
this.pubspec,
this.readme,
this.license,
this.changelog,
this.example,
this.monoPkg,
);
}
/// Configuration of a set of packages generated from a set of APIs exposed by
/// a Discovery Service.
class DiscoveryPackagesConfiguration {
String configFile;
late Map yaml;
late Map<String, Package> packages;
late final Set<String> excessApis;
late final List<String> missingApis;
late final Set<String> skipTests;
final existingApiRevisions = <String, String>{};
Map<String?, String>? newRevisions;
Map<String?, String>? oldRevisions;
/// Create a new discovery packages configuration.
///
/// The format of a YAML document describing a number of packages looks
/// like this:
///
/// packages:
/// googleapis:
/// version: 0.1.0
/// repository: https://github.com/dart-lang/googleapis
/// readme: resources/README.md
/// license: resources/LICENSE
/// apis:
/// - analytics:v3
/// - bigquery:v2
/// googleapis_beta:
/// version: 0.1.0
/// repository: https://github.com/dart-lang/googleapis
/// readme: resources/README.md
/// license: resources/LICENSE
/// apis:
/// - datastore:v1beta2
/// - dns:v1beta1
/// skipped_apis:
/// - adexchangebuyer:v1
///
/// Each package to build is listed under the key `packages:`.
///
/// The key `skipped_apis` is used to list APIs returned buy the Discovery
/// Service but is not part of any generated packages.
///
/// The file names for the content of readme and license files are resolved
/// relative to the configuration file.
DiscoveryPackagesConfiguration(this.configFile) {
yaml = loadYaml(File(configFile).readAsStringSync()) as Map;
}
/// Downloads discovery documents from the configuration.
///
/// [discoveryDocsDir] is the directory where all the downloaded discovery
/// documents are stored.
Future<void> download(String discoveryDocsDir) async {
// Delete all downloaded discovery documents.
final dir = Directory(discoveryDocsDir);
if (dir.existsSync()) {
print('*** Cataloging and deleting existing discovery JSON files');
for (var file in dir.listSync(recursive: true).whereType<File>()) {
final json =
jsonDecode(file.readAsStringSync()) as Map<String, dynamic>;
final id = json['id'] as String;
final revision = json['revision'] as String;
assert(!existingApiRevisions.containsKey(id));
existingApiRevisions[id] = revision;
}
print('Existing API count: ${existingApiRevisions.length}');
dir.deleteSync(recursive: true);
}
// Get all rest discovery documents & initialize this object.
final allApis = await const FetchCore.github().fetchDiscoveryDocuments(
existingRevisions: existingApiRevisions,
);
_initialize(allApis);
if (updateConfigurationFile()) {
_initialize(allApis);
}
var count = 0;
for (var entry in packages.entries) {
print(' ${++count} of ${packages.length} - ${entry.key}');
writeDiscoveryDocuments(
'$discoveryDocsDir/${entry.key}',
entry.value.apis
.map((e) {
final allMatches = allApis
.where((element) => element.id == e)
.toList();
if (allMatches.length == 1) {
return allMatches.single;
}
print(
'Looking 1 match for "$e" - instead found ${allMatches.length}',
);
return null;
})
.whereType<RestDescription>()
.toList(growable: false),
);
}
}
bool updateConfigurationFile() {
final editor = YamlEditor(File(configFile).readAsStringSync());
var changed = false;
if (missingApis.isNotEmpty) {
for (var apiId in missingApis) {
final version = apiId.split(':').last;
final isStable =
!version.contains('alpha') && !version.contains('beta');
final path = isStable
? ['packages', 'googleapis', 'apis']
: ['skipped_apis'];
final list = (editor.parseAt(path) as YamlList).cast<String>().toList();
list.add(apiId);
list.sort();
editor.update(path, list);
changed = true;
}
}
if (excessApis.isNotEmpty) {
for (var apiId in excessApis) {
// Find which list it's in.
final googleApisPath = ['packages', 'googleapis', 'apis'];
final googleApisBetaPath = ['packages', 'googleapis_beta', 'apis'];
final skippedApisPath = ['skipped_apis'];
for (final path in [
googleApisPath,
googleApisBetaPath,
skippedApisPath,
]) {
try {
final node = editor.parseAt(path);
if (node is YamlList) {
final list = node.cast<String>().toList();
if (list.remove(apiId)) {
editor.update(path, list);
changed = true;
}
}
} catch (_) {
// Path might not exist in this config, which is fine.
}
}
}
}
if (changed) {
final newContent = editor.toString();
File(configFile).writeAsStringSync(newContent);
yaml = loadYaml(newContent) as Map;
print('Updated $configFile');
}
return changed;
}
/// Generate packages from the configuration.
///
/// [discoveryDocsDir] is the directory where all the downloaded discovery
/// documents are stored.
///
/// [generatedApisDir] is the directory where the packages are generated.
/// Each package is generated in a sub-directory.
void generate(
String discoveryDocsDir,
String generatedApisDir,
bool deleteExisting,
) {
// Delete all downloaded discovery documents.
final dir = Directory(discoveryDocsDir);
if (!dir.existsSync()) {
throw Exception(
'Error: The given `$discoveryDocsDir` directory does not exist.',
);
}
// Load discovery documents from disc & initialize this object.
final allApis = <RestDescription>[];
for (var name in (yaml['packages'] as Map).keys) {
allApis.addAll(loadDiscoveryDocuments('$discoveryDocsDir/$name'));
}
_initialize(allApis);
// Generate packages.
for (var entry in packages.entries) {
final name = entry.key;
final package = entry.value;
final results = generateAllLibraries(
'$discoveryDocsDir/$name',
'$generatedApisDir/$name',
package.pubspec,
deleteExisting: deleteExisting,
skipTests: skipTests,
);
for (final result in results) {
if (!result.success) {
print(result.toString());
}
}
File(
'$generatedApisDir/$name/README.md',
).writeAsStringSync(package.readme);
File(
'$generatedApisDir/$name/LICENSE',
).writeAsStringSync(package.license);
File(
'$generatedApisDir/$name/CHANGELOG.md',
).writeAsStringSync(package.changelog);
if (package.monoPkg != null) {
File(
'$generatedApisDir/$name/mono_pkg.yaml',
).writeAsStringSync(package.monoPkg!);
}
if (package.example != null) {
Directory('$generatedApisDir/$name/example').createSync();
File(
'$generatedApisDir/$name/example/main.dart',
).writeAsStringSync(package.example!);
}
}
}
/// Initializes the missingApis/excessApis/packages properties from a list
/// of [RestDescription]s.
void _initialize(List<RestDescription> allApis) {
packages = _packagesFromYaml(
yaml['packages'] as YamlMap,
configFile,
allApis,
);
skipTests = _listFromYaml(yaml['skip_tests'] as YamlList?).toSet();
final knownApis = _calculateKnownApis(
packages,
_listFromYaml(yaml['skipped_apis'] as YamlList?),
);
missingApis = _calculateMissingApis(knownApis, allApis);
excessApis = _calculateExcessApis(knownApis, allApis);
if (existingApiRevisions.isNotEmpty) {
for (var api in allApis) {
final existingRevision = existingApiRevisions[api.id!];
if (existingRevision != null) {
final compare = api.revision!.compareTo(existingRevision);
if (compare == 0) {
continue;
}
final value = 'previous: $existingRevision; current: ${api.revision}';
if (compare.isNegative) {
(oldRevisions ??= {})[api.id] = value;
}
if (compare > 0) {
(newRevisions ??= {})[api.id] = value;
}
}
}
}
}
// Return empty list for YAML null value.
static List<String> _listFromYaml(YamlList? value) =>
value?.cast<String>() ?? [];
static String _generateReadme(
String? readmeFile,
String packageName,
List<RestDescription?> items, {
String? packageVersion,
}) {
final sb = StringBuffer();
if (readmeFile != null) {
sb.write(File(readmeFile).readAsStringSync());
}
sb.writeln('''
## Available Google APIs
The following is a list of APIs that are currently available inside this
package.
''');
for (var item in items) {
sb.write('#### ');
if (item!.icons != null &&
item.icons!.x16 != null &&
item.icons!.x16!.startsWith('https://')) {
sb.write('![Logo](${item.icons!.x16}) ');
}
final libraryName = ApiLibraryNamer.libraryName(item.name, item.version);
sb
..writeln('${item.title} - `$libraryName`')
..writeln();
if (item.description != null && item.description!.isNotEmpty) {
sb
..writeln(item.description)
..writeln();
}
if (item.documentationLink != null) {
sb.writeln('- [Original documentation](${item.documentationLink})');
}
if (packageVersion != null) {
final libraryPathSection = libraryName.replaceAll('/', '_');
final basePubUri =
'https://pub.dev/documentation/$packageName/$packageVersion/';
final pubUri =
'$basePubUri$libraryPathSection/$libraryPathSection-library.html';
sb.writeln('- [Dart package details]($pubUri)');
}
sb.writeln();
}
return sb.toString();
}
static Map<String, Package> _packagesFromYaml(
YamlMap configPackages,
String configFile,
List<RestDescription?> allApis,
) {
final packages = <String, Package>{};
configPackages.forEach((name, values) {
packages[name as String] = _packageFromYaml(
name,
values as YamlMap,
configFile,
allApis,
);
});
return packages;
}
static Package _packageFromYaml(
String name,
YamlMap values,
String configFile,
List<RestDescription?> allApis,
) {
final apis = _listFromYaml(values['apis'] as YamlList).cast<String>();
final version = values['version'] as String?;
final author = values['author'] as String?;
final repository = values['repository'] as String?;
final resolution = values['resolution'] as String?;
Map<String, String>? extraDevDependencies;
if (values.containsKey('extraDevDependencies')) {
extraDevDependencies = (values['extraDevDependencies'] as YamlMap)
.cast<String, String>();
}
final configUri = Uri.file(configFile);
allApis.sort((RestDescription? a, RestDescription? b) {
final result = a!.name!.compareTo(b!.name!);
if (result != 0) return result;
return a.version!.compareTo(b.version!);
});
String? readmeFile;
if (values['readme'] != null) {
readmeFile = configUri.resolve(values['readme'] as String).path;
}
late String licenseFile;
if (values['license'] != null) {
licenseFile = configUri.resolve(values['license'] as String).path;
}
late String changelogFile;
if (values['changelog'] != null) {
changelogFile = configUri.resolve(values['changelog'] as String).path;
}
String? example;
if (values['example'] != null) {
final exampleFile = configUri.resolve(values['example'] as String).path;
example = File(exampleFile).readAsStringSync();
}
String? monoPkg;
if (values['mono_pkg'] != null) {
final monoPkgFile = configUri.resolve(values['mono_pkg'] as String).path;
monoPkg = File(monoPkgFile).readAsStringSync();
}
// Generate package description.
final apiDescriptions = <RestDescription?>[];
const description =
'Auto-generated client libraries for accessing Google '
'APIs described through the API discovery service.';
for (var apiDescription in allApis) {
if (apis.contains(apiDescription!.id)) {
apiDescriptions.add(apiDescription);
}
}
// Generate the README.md file content.
final readme = _generateReadme(
readmeFile,
name,
apiDescriptions,
packageVersion: version,
);
// Read the LICENSE
final license = File(licenseFile).readAsStringSync();
// Read CHANGELOG.md
final changelog = File(changelogFile).readAsStringSync();
// Create package description with pubspec.yaml information.
final pubspec = Pubspec(
name,
version,
description,
author: author,
repository: repository,
extraDevDependencies: extraDevDependencies,
resolution: resolution,
);
return Package(
name,
apis,
pubspec,
readme,
license,
changelog,
example,
monoPkg,
);
}
/// The known APIs are the APis mentioned in each package together with
/// the APIs explicitly skipped.
static Set<String> _calculateKnownApis(
Map<String, Package> packages,
List<String> skippedApis,
) => <String>{...skippedApis, ...packages.values.expand((v) => v.apis)};
/// The missing APIs are the APIs returned from the Discovery Service
/// but not mentioned in the configuration.
static List<String> _calculateMissingApis(
Iterable<String> knownApis,
List<RestDescription> allApis,
) => allApis
.where((item) => !knownApis.contains(item.id))
.map((item) => item.id!)
.toList();
/// The excess APIs are the APIs mentioned in the configuration but not
/// returned from the Discovery Service.
static Set<String> _calculateExcessApis(
Iterable<String> knownApis,
List<RestDescription> allApis,
) => Set<String>.from(knownApis)..removeAll(allApis.map((e) => e.id));
}