| // Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:convert'; |
| import 'dart:io'; |
| |
| import 'package:discoveryapis_generator/discoveryapis_generator.dart'; |
| // ignore: implementation_imports |
| import 'package:discoveryapis_generator/src/namer.dart'; |
| import 'package:yaml/yaml.dart'; |
| import 'package:yaml_edit/yaml_edit.dart'; |
| |
| import '../googleapis_generator.dart'; |
| import 'fetch_core.dart'; |
| |
| class Package { |
| final String name; |
| final List<String> apis; |
| final Pubspec pubspec; |
| final String readme; |
| final String license; |
| final String changelog; |
| final String? example; |
| final String? monoPkg; |
| |
| Package( |
| this.name, |
| this.apis, |
| this.pubspec, |
| this.readme, |
| this.license, |
| this.changelog, |
| this.example, |
| this.monoPkg, |
| ); |
| } |
| |
| /// Configuration of a set of packages generated from a set of APIs exposed by |
| /// a Discovery Service. |
| class DiscoveryPackagesConfiguration { |
| String configFile; |
| late Map yaml; |
| late Map<String, Package> packages; |
| |
| late final Set<String> excessApis; |
| late final List<String> missingApis; |
| late final Set<String> skipTests; |
| final existingApiRevisions = <String, String>{}; |
| Map<String?, String>? newRevisions; |
| Map<String?, String>? oldRevisions; |
| |
| /// Create a new discovery packages configuration. |
| /// |
| /// The format of a YAML document describing a number of packages looks |
| /// like this: |
| /// |
| /// packages: |
| /// googleapis: |
| /// version: 0.1.0 |
| /// repository: https://github.com/dart-lang/googleapis |
| /// readme: resources/README.md |
| /// license: resources/LICENSE |
| /// apis: |
| /// - analytics:v3 |
| /// - bigquery:v2 |
| /// googleapis_beta: |
| /// version: 0.1.0 |
| /// repository: https://github.com/dart-lang/googleapis |
| /// readme: resources/README.md |
| /// license: resources/LICENSE |
| /// apis: |
| /// - datastore:v1beta2 |
| /// - dns:v1beta1 |
| /// skipped_apis: |
| /// - adexchangebuyer:v1 |
| /// |
| /// Each package to build is listed under the key `packages:`. |
| /// |
| /// The key `skipped_apis` is used to list APIs returned buy the Discovery |
| /// Service but is not part of any generated packages. |
| /// |
| /// The file names for the content of readme and license files are resolved |
| /// relative to the configuration file. |
| DiscoveryPackagesConfiguration(this.configFile) { |
| yaml = loadYaml(File(configFile).readAsStringSync()) as Map; |
| } |
| |
| /// Downloads discovery documents from the configuration. |
| /// |
| /// [discoveryDocsDir] is the directory where all the downloaded discovery |
| /// documents are stored. |
| Future<void> download(String discoveryDocsDir) async { |
| // Delete all downloaded discovery documents. |
| final dir = Directory(discoveryDocsDir); |
| if (dir.existsSync()) { |
| print('*** Cataloging and deleting existing discovery JSON files'); |
| for (var file in dir.listSync(recursive: true).whereType<File>()) { |
| final json = |
| jsonDecode(file.readAsStringSync()) as Map<String, dynamic>; |
| final id = json['id'] as String; |
| final revision = json['revision'] as String; |
| assert(!existingApiRevisions.containsKey(id)); |
| existingApiRevisions[id] = revision; |
| } |
| |
| print('Existing API count: ${existingApiRevisions.length}'); |
| |
| dir.deleteSync(recursive: true); |
| } |
| |
| // Get all rest discovery documents & initialize this object. |
| final allApis = await const FetchCore.github().fetchDiscoveryDocuments( |
| existingRevisions: existingApiRevisions, |
| ); |
| _initialize(allApis); |
| |
| if (updateConfigurationFile()) { |
| _initialize(allApis); |
| } |
| |
| var count = 0; |
| for (var entry in packages.entries) { |
| print(' ${++count} of ${packages.length} - ${entry.key}'); |
| |
| writeDiscoveryDocuments( |
| '$discoveryDocsDir/${entry.key}', |
| entry.value.apis |
| .map((e) { |
| final allMatches = allApis |
| .where((element) => element.id == e) |
| .toList(); |
| |
| if (allMatches.length == 1) { |
| return allMatches.single; |
| } |
| |
| print( |
| 'Looking 1 match for "$e" - instead found ${allMatches.length}', |
| ); |
| return null; |
| }) |
| .whereType<RestDescription>() |
| .toList(growable: false), |
| ); |
| } |
| } |
| |
| bool updateConfigurationFile() { |
| final editor = YamlEditor(File(configFile).readAsStringSync()); |
| |
| var changed = false; |
| |
| if (missingApis.isNotEmpty) { |
| for (var apiId in missingApis) { |
| final version = apiId.split(':').last; |
| final isStable = |
| !version.contains('alpha') && !version.contains('beta'); |
| |
| final path = isStable |
| ? ['packages', 'googleapis', 'apis'] |
| : ['skipped_apis']; |
| |
| final list = (editor.parseAt(path) as YamlList).cast<String>().toList(); |
| list.add(apiId); |
| list.sort(); |
| editor.update(path, list); |
| changed = true; |
| } |
| } |
| |
| if (excessApis.isNotEmpty) { |
| for (var apiId in excessApis) { |
| // Find which list it's in. |
| final googleApisPath = ['packages', 'googleapis', 'apis']; |
| final googleApisBetaPath = ['packages', 'googleapis_beta', 'apis']; |
| final skippedApisPath = ['skipped_apis']; |
| |
| for (final path in [ |
| googleApisPath, |
| googleApisBetaPath, |
| skippedApisPath, |
| ]) { |
| try { |
| final node = editor.parseAt(path); |
| if (node is YamlList) { |
| final list = node.cast<String>().toList(); |
| if (list.remove(apiId)) { |
| editor.update(path, list); |
| changed = true; |
| } |
| } |
| } catch (_) { |
| // Path might not exist in this config, which is fine. |
| } |
| } |
| } |
| } |
| |
| if (changed) { |
| final newContent = editor.toString(); |
| File(configFile).writeAsStringSync(newContent); |
| yaml = loadYaml(newContent) as Map; |
| print('Updated $configFile'); |
| } |
| |
| return changed; |
| } |
| |
| /// Generate packages from the configuration. |
| /// |
| /// [discoveryDocsDir] is the directory where all the downloaded discovery |
| /// documents are stored. |
| /// |
| /// [generatedApisDir] is the directory where the packages are generated. |
| /// Each package is generated in a sub-directory. |
| void generate( |
| String discoveryDocsDir, |
| String generatedApisDir, |
| bool deleteExisting, |
| ) { |
| // Delete all downloaded discovery documents. |
| final dir = Directory(discoveryDocsDir); |
| if (!dir.existsSync()) { |
| throw Exception( |
| 'Error: The given `$discoveryDocsDir` directory does not exist.', |
| ); |
| } |
| |
| // Load discovery documents from disc & initialize this object. |
| final allApis = <RestDescription>[]; |
| for (var name in (yaml['packages'] as Map).keys) { |
| allApis.addAll(loadDiscoveryDocuments('$discoveryDocsDir/$name')); |
| } |
| _initialize(allApis); |
| |
| // Generate packages. |
| for (var entry in packages.entries) { |
| final name = entry.key; |
| final package = entry.value; |
| final results = generateAllLibraries( |
| '$discoveryDocsDir/$name', |
| '$generatedApisDir/$name', |
| package.pubspec, |
| deleteExisting: deleteExisting, |
| skipTests: skipTests, |
| ); |
| for (final result in results) { |
| if (!result.success) { |
| print(result.toString()); |
| } |
| } |
| |
| File( |
| '$generatedApisDir/$name/README.md', |
| ).writeAsStringSync(package.readme); |
| File( |
| '$generatedApisDir/$name/LICENSE', |
| ).writeAsStringSync(package.license); |
| File( |
| '$generatedApisDir/$name/CHANGELOG.md', |
| ).writeAsStringSync(package.changelog); |
| if (package.monoPkg != null) { |
| File( |
| '$generatedApisDir/$name/mono_pkg.yaml', |
| ).writeAsStringSync(package.monoPkg!); |
| } |
| if (package.example != null) { |
| Directory('$generatedApisDir/$name/example').createSync(); |
| File( |
| '$generatedApisDir/$name/example/main.dart', |
| ).writeAsStringSync(package.example!); |
| } |
| } |
| } |
| |
| /// Initializes the missingApis/excessApis/packages properties from a list |
| /// of [RestDescription]s. |
| void _initialize(List<RestDescription> allApis) { |
| packages = _packagesFromYaml( |
| yaml['packages'] as YamlMap, |
| configFile, |
| allApis, |
| ); |
| skipTests = _listFromYaml(yaml['skip_tests'] as YamlList?).toSet(); |
| final knownApis = _calculateKnownApis( |
| packages, |
| _listFromYaml(yaml['skipped_apis'] as YamlList?), |
| ); |
| missingApis = _calculateMissingApis(knownApis, allApis); |
| excessApis = _calculateExcessApis(knownApis, allApis); |
| |
| if (existingApiRevisions.isNotEmpty) { |
| for (var api in allApis) { |
| final existingRevision = existingApiRevisions[api.id!]; |
| if (existingRevision != null) { |
| final compare = api.revision!.compareTo(existingRevision); |
| if (compare == 0) { |
| continue; |
| } |
| final value = 'previous: $existingRevision; current: ${api.revision}'; |
| if (compare.isNegative) { |
| (oldRevisions ??= {})[api.id] = value; |
| } |
| if (compare > 0) { |
| (newRevisions ??= {})[api.id] = value; |
| } |
| } |
| } |
| } |
| } |
| |
| // Return empty list for YAML null value. |
| static List<String> _listFromYaml(YamlList? value) => |
| value?.cast<String>() ?? []; |
| |
| static String _generateReadme( |
| String? readmeFile, |
| String packageName, |
| List<RestDescription?> items, { |
| String? packageVersion, |
| }) { |
| final sb = StringBuffer(); |
| if (readmeFile != null) { |
| sb.write(File(readmeFile).readAsStringSync()); |
| } |
| sb.writeln(''' |
| |
| ## Available Google APIs |
| |
| The following is a list of APIs that are currently available inside this |
| package. |
| '''); |
| |
| for (var item in items) { |
| sb.write('#### '); |
| if (item!.icons != null && |
| item.icons!.x16 != null && |
| item.icons!.x16!.startsWith('https://')) { |
| sb.write(' '); |
| } |
| final libraryName = ApiLibraryNamer.libraryName(item.name, item.version); |
| sb |
| ..writeln('${item.title} - `$libraryName`') |
| ..writeln(); |
| |
| if (item.description != null && item.description!.isNotEmpty) { |
| sb |
| ..writeln(item.description) |
| ..writeln(); |
| } |
| |
| if (item.documentationLink != null) { |
| sb.writeln('- [Original documentation](${item.documentationLink})'); |
| } |
| |
| if (packageVersion != null) { |
| final libraryPathSection = libraryName.replaceAll('/', '_'); |
| final basePubUri = |
| 'https://pub.dev/documentation/$packageName/$packageVersion/'; |
| final pubUri = |
| '$basePubUri$libraryPathSection/$libraryPathSection-library.html'; |
| sb.writeln('- [Dart package details]($pubUri)'); |
| } |
| sb.writeln(); |
| } |
| return sb.toString(); |
| } |
| |
| static Map<String, Package> _packagesFromYaml( |
| YamlMap configPackages, |
| String configFile, |
| List<RestDescription?> allApis, |
| ) { |
| final packages = <String, Package>{}; |
| configPackages.forEach((name, values) { |
| packages[name as String] = _packageFromYaml( |
| name, |
| values as YamlMap, |
| configFile, |
| allApis, |
| ); |
| }); |
| |
| return packages; |
| } |
| |
| static Package _packageFromYaml( |
| String name, |
| YamlMap values, |
| String configFile, |
| List<RestDescription?> allApis, |
| ) { |
| final apis = _listFromYaml(values['apis'] as YamlList).cast<String>(); |
| final version = values['version'] as String?; |
| final author = values['author'] as String?; |
| final repository = values['repository'] as String?; |
| final resolution = values['resolution'] as String?; |
| |
| Map<String, String>? extraDevDependencies; |
| if (values.containsKey('extraDevDependencies')) { |
| extraDevDependencies = (values['extraDevDependencies'] as YamlMap) |
| .cast<String, String>(); |
| } |
| |
| final configUri = Uri.file(configFile); |
| |
| allApis.sort((RestDescription? a, RestDescription? b) { |
| final result = a!.name!.compareTo(b!.name!); |
| if (result != 0) return result; |
| return a.version!.compareTo(b.version!); |
| }); |
| |
| String? readmeFile; |
| if (values['readme'] != null) { |
| readmeFile = configUri.resolve(values['readme'] as String).path; |
| } |
| late String licenseFile; |
| if (values['license'] != null) { |
| licenseFile = configUri.resolve(values['license'] as String).path; |
| } |
| late String changelogFile; |
| if (values['changelog'] != null) { |
| changelogFile = configUri.resolve(values['changelog'] as String).path; |
| } |
| String? example; |
| if (values['example'] != null) { |
| final exampleFile = configUri.resolve(values['example'] as String).path; |
| example = File(exampleFile).readAsStringSync(); |
| } |
| |
| String? monoPkg; |
| if (values['mono_pkg'] != null) { |
| final monoPkgFile = configUri.resolve(values['mono_pkg'] as String).path; |
| monoPkg = File(monoPkgFile).readAsStringSync(); |
| } |
| |
| // Generate package description. |
| final apiDescriptions = <RestDescription?>[]; |
| const description = |
| 'Auto-generated client libraries for accessing Google ' |
| 'APIs described through the API discovery service.'; |
| for (var apiDescription in allApis) { |
| if (apis.contains(apiDescription!.id)) { |
| apiDescriptions.add(apiDescription); |
| } |
| } |
| |
| // Generate the README.md file content. |
| final readme = _generateReadme( |
| readmeFile, |
| name, |
| apiDescriptions, |
| packageVersion: version, |
| ); |
| |
| // Read the LICENSE |
| final license = File(licenseFile).readAsStringSync(); |
| |
| // Read CHANGELOG.md |
| final changelog = File(changelogFile).readAsStringSync(); |
| |
| // Create package description with pubspec.yaml information. |
| final pubspec = Pubspec( |
| name, |
| version, |
| description, |
| author: author, |
| repository: repository, |
| extraDevDependencies: extraDevDependencies, |
| resolution: resolution, |
| ); |
| return Package( |
| name, |
| apis, |
| pubspec, |
| readme, |
| license, |
| changelog, |
| example, |
| monoPkg, |
| ); |
| } |
| |
| /// The known APIs are the APis mentioned in each package together with |
| /// the APIs explicitly skipped. |
| static Set<String> _calculateKnownApis( |
| Map<String, Package> packages, |
| List<String> skippedApis, |
| ) => <String>{...skippedApis, ...packages.values.expand((v) => v.apis)}; |
| |
| /// The missing APIs are the APIs returned from the Discovery Service |
| /// but not mentioned in the configuration. |
| static List<String> _calculateMissingApis( |
| Iterable<String> knownApis, |
| List<RestDescription> allApis, |
| ) => allApis |
| .where((item) => !knownApis.contains(item.id)) |
| .map((item) => item.id!) |
| .toList(); |
| |
| /// The excess APIs are the APIs mentioned in the configuration but not |
| /// returned from the Discovery Service. |
| static Set<String> _calculateExcessApis( |
| Iterable<String> knownApis, |
| List<RestDescription> allApis, |
| ) => Set<String>.from(knownApis)..removeAll(allApis.map((e) => e.id)); |
| } |