packages/flutter_tools/lib/src/commands/update_packages.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';
import 'dart:collection';

import 'package:pub_semver/pub_semver.dart';
import 'package:pubspec_parse/pubspec_parse.dart';
import 'package:yaml/yaml.dart';
import 'package:yaml_edit/yaml_edit.dart';

import '../base/common.dart';
import '../base/context.dart';
import '../base/file_system.dart';
import '../base/net.dart';
import '../cache.dart';
import '../context/tool_context.dart';
import '../dart/pub.dart';
import '../globals.dart' as globals;
import '../project.dart';
import '../runner/flutter_command.dart';
import '../update_packages_pins.dart';

// Pub packages are rolled automatically by the flutter-pub-roller-bot
// by using the `flutter update-packages --force-upgrade`.
// For the latest status, see:
//   https://github.com/pulls?q=author%3Aflutter-pub-roller-bot

const _pubspecName = 'pubspec.yaml';

typedef _ProjectDeps = ({FlutterProject project, ResolvedDependencies deps});

/// A command to update internal dependencies across Flutter packages.
///
/// Note: Full internal migration away from ambient `globals.*` in this command
/// is deferred until additional services (such as HTTP clients) are integrated
/// into [ToolContext].
class UpdatePackagesCommand extends FlutterCommand {
  UpdatePackagesCommand({required ToolContext toolContext, required bool verboseHelp})
    : super(toolContext: toolContext, verboseHelp: verboseHelp) {
    argParser
      ..addFlag(
        _keyForceUpgrade,
        help:
            'Attempt to update all the dependencies to their latest versions.\n'
            'This will actually modify the pubspec.yaml files in your checkout.',
        negatable: false,
      )
      ..addFlag(
        _keyUpdateHashes,
        help: 'Update the hashes of the pubspecs.',
        negatable: false,
        // We don't want to promote usage, to not circumvent using this script to update
        hide: !verboseHelp,
      )
      ..addMultiOption(
        _keyCherryPick,
        help: 'Attempt to update only the specified package. To be specified as [pub package name]:[pub package version],[pub package2 name]:[pub package2 version].',
      )
      ..addFlag(
        _keyOffline,
        help: 'Use cached packages instead of accessing the network.',
        negatable: false,
      )
      ..addFlag(
        _keyUpgradeMajor,
        help: 'Upgrade major versions as well. Only makes sense with force-upgrade.',
      )
      ..addFlag(
        _keyExcludeTools,
        help: "Don't update the deps in tools. For example when unpinning a dep.",
      )
      ..addFlag(
        _keyCrash,
        help: 'For Flutter CLI testing only, forces this command to throw an unhandled exception.',
        negatable: false,
        hide: !verboseHelp,
      );
  }

  final _keyForceUpgrade = 'force-upgrade';
  final _keyUpdateHashes = 'update-hashes';
  final _keyCherryPick = 'cherry-pick';
  final _keyOffline = 'offline';
  final _keyUpgradeMajor = 'upgrade-major';
  final _keyExcludeTools = 'exclude-tools';
  final _keyCrash = 'crash';

  static const fixedPackages = <String>{'test_api', 'test_core'};

  @override
  final name = 'update-packages';

  @override
  final description =
      'Update the packages inside the Flutter repo. '
      'This is intended for CI and repo maintainers. '
      'Normal Flutter developers should not have to '
      'use this command.';

  @override
  final aliases = <String>['upgrade-packages'];

  @override
  final hidden = true;

  // Lazy-initialize the net utilities with values from the context.
  late final _net = Net(
    httpClientFactory: context.get<HttpClientFactory>(),
    logger: globals.logger,
    platform: globals.platform,
  );

  Future<void> _downloadCoverageData() async {
    final String urlBase =
        globals.platform.environment[kFlutterStorageBaseUrl] ?? 'https://storage.googleapis.com';
    final Uri coverageUri = Uri.parse('$urlBase/flutter_infra_release/flutter/coverage/lcov.info');
    final List<int>? data = await _net.fetchUrl(coverageUri, maxAttempts: 3);
    if (data == null) {
      throwToolExit('Failed to fetch coverage data from $coverageUri');
    }
    final String coverageDir = globals.fs.path.join(
      Cache.flutterRoot!,
      'packages/flutter/coverage',
    );
    globals.fs.file(globals.fs.path.join(coverageDir, 'lcov.base.info'))
      ..createSync(recursive: true)
      ..writeAsBytesSync(data, flush: true);
    globals.fs.file(globals.fs.path.join(coverageDir, 'lcov.info'))
      ..createSync(recursive: true)
      ..writeAsBytesSync(data, flush: true);
  }

  @override
  Future<FlutterCommandResult> runCommand() async {
    // Add the root directory to the list of packages, to capture the workspace
    // `pubspec.yaml`.
    final Directory rootDirectory = globals.fs.directory(
      globals.fs.path.absolute(Cache.flutterRoot!),
    );

    final bool forceUpgrade = boolArg(_keyForceUpgrade);
    final bool updateHashes = boolArg(_keyUpdateHashes);
    final bool offline = boolArg(_keyOffline);
    final List<PackageVersion> cherryPicks = stringsArg(_keyCherryPick)
        .map((String e) => e.split(':'))
        .map((List<String> e) => (package: e[0], version: e[1]))
        .toList();
    final bool relaxToAny = boolArg(_keyUpgradeMajor);
    final bool excludeTools = boolArg(_keyExcludeTools);

    if (boolArg('crash')) {
      throw StateError('test crash please ignore.');
    }

    if (forceUpgrade && offline) {
      throwToolExit('--force-upgrade cannot be used with the --offline flag');
    }

    if (forceUpgrade && cherryPicks.isNotEmpty) {
      throwToolExit('--force-upgrade cannot be used with the --cherry-pick-package flag');
    }

    if (cherryPicks.isNotEmpty && offline) {
      throwToolExit('--cherry-pick-package cannot be used with the --offline flag');
    }

    if (forceUpgrade) {
      // This feature attempts to collect all the packages used across all the
      // pubspec.yamls in the repo (including via transitive dependencies), and
      // find the latest version of each that can be used while keeping each
      // such package fixed at a single version across all the pubspec.yamls.
      globals.printStatus('Upgrading packages...');
    }
    final FlutterProject rootProject = FlutterProject.fromDirectory(rootDirectory);
    final FlutterProject toolProject = FlutterProject.fromDirectory(
      rootDirectory.childDirectory('packages').childDirectory('flutter_tools'),
    );

    // This package is intentionally not part of the workspace to test
    // user-defines in its local pubspec.
    final Directory hooksUserDefineIntegrationTestDirectory = rootDirectory
        .childDirectory('dev')
        .childDirectory('integration_tests')
        .childDirectory('hook_user_defines');

    final packages = <Directory>[...runner!.getRepoPackages(), rootDirectory];

    if (!updateHashes) {
      _verifyPubspecs(packages);
    }
    if (forceUpgrade || cherryPicks.isNotEmpty) {
      final (project: _, :ResolvedDependencies deps) = (await _upgrade(
        forceUpgrade: forceUpgrade,
        cherryPicks: cherryPicks,
        projects: [rootProject],
        relaxToAny: relaxToAny,
      )).single;

      for (final package in <Directory>[
        rootDirectory,
        rootDirectory.childDirectory('packages').childDirectory('flutter'),
        rootDirectory.childDirectory('packages').childDirectory('flutter_test'),
        rootDirectory.childDirectory('packages').childDirectory('flutter_localizations'),
        hooksUserDefineIntegrationTestDirectory,
      ]) {
        _updatePubspec(package, deps);
      }

      if (!excludeTools) {
        final List<_ProjectDeps> toolDeps = await _upgrade(
          forceUpgrade: forceUpgrade,
          cherryPicks: cherryPicks,
          // Ensure that flutter_tools uses the same versions for packages that are also
          // used by the Flutter SDK.
          pinned: deps.toVersions(),
          projects: <FlutterProject>[toolProject],
          relaxToAny: relaxToAny,
        );
        for (final (:project, :deps) in toolDeps) {
          _updatePubspec(project.directory, deps);
          // When flutter_tools contains sub-packages in its workspace (under packages/),
          // also update their pubspecs with the resolved dependencies so that shared
          // package constraints (e.g. meta) remain in sync across the workspace.
          final Directory subpackagesDir = project.directory.childDirectory('packages');
          if (subpackagesDir.existsSync()) {
            for (final FileSystemEntity entity in subpackagesDir.listSync()) {
              if (entity is Directory && entity.childFile(_pubspecName).existsSync()) {
                _updatePubspec(entity, deps);
              }
            }
          }
        }
      }
    }
    globals.printStatus('Running pub get only...');
    if (updateHashes || forceUpgrade || cherryPicks.isNotEmpty) {
      _writeHashesToPubspecs(packages);
    }
    _verifyPubspecs(packages);
    _checkWithFlutterTools(rootDirectory);
    _checkPins(rootDirectory);

    // Pub get for the workspace.
    await _pubGet(rootProject, !forceUpgrade && cherryPicks.isEmpty && !updateHashes);

    // Manually do a pub get for packages not part of the workspace.
    // See https://github.com/flutter/flutter/pull/170364.
    await _pubGet(toolProject, false);
    await _pubGet(FlutterProject.fromDirectory(hooksUserDefineIntegrationTestDirectory), false);

    await _downloadCoverageData();

    return FlutterCommandResult.success();
  }

  Future<void> _pubGet(FlutterProject project, bool enforceLockfile) async =>
      pub.get(context: PubContext.pubGet, project: project, enforceLockfile: enforceLockfile);

  Future<List<_ProjectDeps>> _upgrade({
    required bool forceUpgrade,
    required List<PackageVersion> cherryPicks,
    required bool relaxToAny,
    required List<FlutterProject> projects,
    List<PackageVersion>? pinned,
  }) async {
    final Map<String, String> pinnedDeps;
    if (forceUpgrade) {
      globals.printStatus('Upgrading packages versions...');
      pinnedDeps = {...kManuallyPinnedDependencies};
    } else if (cherryPicks.isNotEmpty) {
      globals.printStatus('Pinning packages "$cherryPicks"...');
      pinnedDeps = <String, String>{
        for (final PackageVersion cherryPick in cherryPicks) cherryPick.package: cherryPick.version,
      };
    } else {
      throw StateError('To get here, either forceUpgrade or cherry pick should be set.');
    }

    pinnedDeps.addAll({
      for (final (:package, :version) in pinned ?? <PackageVersion>[]) package: version,
    });

    final Directory tempDir = globals.fs.systemTempDirectory.createTempSync(
      'flutter_upgrade_packages.',
    );
    final deps = <_ProjectDeps>[];
    for (final project in projects) {
      final Directory projectTempDir = tempDir.childDirectory(
        globals.fs.path.relative(project.directory.path, from: Cache.flutterRoot),
      );
      final File tempPubspec = projectTempDir.childFile(project.pubspecFile.basename)
        ..createSync(recursive: true);
      globals.printStatus('Writing to temp pubspec at $tempPubspec');
      final String pubspecContents = project.pubspecFile.readAsStringSync();
      final yamlEditor = YamlEditor(pubspecContents);
      final ResolvedDependencies oldDeps = _fetchDeps(yamlEditor);
      final workspacePath = <String>['workspace'];
      final YamlNode workspaceNode = yamlEditor.parseAt(
        workspacePath,
        orElse: () => wrapAsYamlNode(null),
      );
      final workspaceMembers = <String>{};
      if (workspaceNode is YamlList) {
        for (final Object? member in workspaceNode) {
          if (member is String) {
            String memberName = globals.fs.path.basename(member);
            final File memberPubspec = globals.fs.file(
              globals.fs.path.join(project.directory.path, member, _pubspecName),
            );
            if (memberPubspec.existsSync()) {
              try {
                memberName = Pubspec.parse(memberPubspec.readAsStringSync()).name;
              } on Exception {
                // Fall back to basename if parsing fails.
              }
            }
            workspaceMembers.add(memberName);
          }
        }
        yamlEditor.remove(workspacePath);
      }
      final RelaxMode relaxMode = switch (cherryPicks.isNotEmpty) {
        true => RelaxMode.strict,
        false => relaxToAny ? RelaxMode.any : RelaxMode.caret,
      };
      _relaxDeps(yamlEditor, relaxMode, pinnedDeps);
      _removePathAndWorkspaceDependencies(yamlEditor, project.directory, workspaceMembers);
      tempPubspec.writeAsStringSync(yamlEditor.toString());
      globals.printStatus('Upgrade in $projectTempDir (for project: ${project.manifest.appName})');
      await pub.interactively(
        <String>['upgrade', '--tighten', '-C', projectTempDir.path],
        context: PubContext.updatePackages,
        project: FlutterProject.fromDirectory(projectTempDir),
        command: 'update',
      );

      final ResolvedDependencies newDeps = _fetchDeps(YamlEditor(tempPubspec.readAsStringSync()));

      deps.add((
        project: project,
        deps: ResolvedDependencies.mergeDeps(oldDeps, newDeps, cherryPicks),
      ));
    }

    tempDir.deleteSync(recursive: true);
    return deps;
  }

  /// Removes local path dependencies and workspace member dependencies from the
  /// temporary pubspec before running `pub upgrade --tighten`.
  ///
  /// `_upgrade` runs `dart pub upgrade` inside an isolated temporary directory
  /// where only the project's root `pubspec.yaml` is written. Workspace members
  /// and local subpackages (e.g. `flutter_tools_core`) do not exist in the
  /// temporary directory. Removing them from the temporary pubspec prevents
  /// version solving failures caused by missing local paths or attempting to
  /// resolve unreleased workspace subpackages from pub.dev. The actual path
  /// dependencies and workspace definitions in the repository are preserved and
  /// not affected by this removal.
  void _removePathAndWorkspaceDependencies(
    YamlEditor yamlEditor,
    Directory projectDirectory,
    Set<String> workspaceMembers,
  ) {
    final Directory subpackagesDir = projectDirectory.childDirectory('packages');
    for (final dependencyType in <String>[
      'dependencies',
      'dev_dependencies',
      'dependency_overrides',
    ]) {
      final YamlNode node = yamlEditor.parseAt(<String>[
        dependencyType,
      ], orElse: () => wrapAsYamlNode(null));
      if (node is! YamlMap) {
        continue;
      }
      final toRemove = <String>[];
      for (final MapEntry<Object?, Object?> dep in node.entries) {
        final Object? value = dep.value;
        final packageName = dep.key! as String;
        final bool isPathDependency = value is Map && value.containsKey('path');
        final bool isWorkspaceMember =
            workspaceMembers.contains(packageName) ||
            (subpackagesDir.existsSync() &&
                subpackagesDir.childDirectory(packageName).childFile(_pubspecName).existsSync());
        if (isPathDependency || (value == null && isWorkspaceMember)) {
          toRemove.add(packageName);
        }
      }
      for (final packageName in toRemove) {
        yamlEditor.remove(<String>[dependencyType, packageName]);
      }
    }
  }

  void _relaxDeps(YamlEditor yamlEditor, RelaxMode relaxMode, Map<String, String> fixedDeps) {
    ResolvedDependencies().forEach(
      yamlEditor: yamlEditor,
      func:
          (Map<String, String> dependencies, String depType, String packageName, Object? version) {
            if (version is String) {
              if (fixedDeps.containsKey(packageName)) {
                yamlEditor.update(<String>[depType, packageName], fixedDeps[packageName]);
              } else {
                yamlEditor.update(
                  <String>[depType, packageName],
                  switch (relaxMode) {
                    RelaxMode.any => 'any',
                    RelaxMode.caret => _versionWithCaret(version),
                    RelaxMode.strict => _versionWithoutCaret(version),
                  },
                );
              }
            }
          },
    );
  }

  ResolvedDependencies _fetchDeps(YamlEditor yamlEditor) {
    return ResolvedDependencies()..forEach(
      yamlEditor: yamlEditor,
      func:
          (Map<String, String> dependencies, String depType, String packageName, Object? version) {
            if (version is String) {
              dependencies[packageName] = version;
            }
          },
    );
  }

  void _updatePubspec(Directory package, ResolvedDependencies dependencies) {
    final File pubspecFile = package.childFile(_pubspecName);
    final yamlEditor = YamlEditor(pubspecFile.readAsStringSync());
    dependencies.forEach(
      yamlEditor: yamlEditor,
      func:
          (Map<String, String> dependencies, String depType, String packageName, Object? version) {
            if (dependencies.containsKey(packageName)) {
              final String version = dependencies[packageName]!;
              yamlEditor.update(<String>[depType, packageName], version);
            }
          },
    );
    pubspecFile.writeAsStringSync(yamlEditor.toString());
  }

  void _verifyPubspecs(List<Directory> packages) {
    globals.printStatus('Verifying pubspecs...');
    for (final directory in packages) {
      globals.printTrace('Reading pubspec.yaml from ${directory.path}');
      final String pubspecString = directory.childFile(_pubspecName).readAsStringSync();
      _checkHash(pubspecString, directory);
    }
  }

  void _checkWithFlutterTools(Directory rootDirectory) {
    final pubspec = Pubspec.parse(rootDirectory.childFile(_pubspecName).readAsStringSync());
    final pubspecTools = Pubspec.parse(
      rootDirectory
          .childDirectory('packages')
          .childDirectory('flutter_tools')
          .childFile(_pubspecName)
          .readAsStringSync(),
    );
    for (final String package in fixedPackages) {
      if (!(pubspec.dependencies[package] == pubspecTools.dependencies[package] &&
          pubspec.devDependencies[package] == pubspecTools.devDependencies[package] &&
          pubspec.dependencyOverrides[package] == pubspecTools.dependencyOverrides[package])) {
        throwToolExit('The dependency on $package must be fixed between flutter and flutter_tools');
      }
    }
  }

  void _checkPins(Directory directory) {
    final pubspec = Pubspec.parse(directory.childFile(_pubspecName).readAsStringSync());
    for (final MapEntry<String, String> pin in kManuallyPinnedDependencies.entries) {
      Dependency dependency;
      if (pubspec.dependencies.containsKey(pin.key)) {
        dependency = pubspec.dependencies[pin.key]!;
      } else if (pubspec.devDependencies.containsKey(pin.key)) {
        dependency = pubspec.devDependencies[pin.key]!;
      } else if (pubspec.dependencyOverrides.containsKey(pin.key)) {
        dependency = pubspec.dependencyOverrides[pin.key]!;
      } else {
        continue;
      }
      final VersionConstraint? version = switch (dependency) {
        SdkDependency(:final VersionConstraint version) ||
        HostedDependency(:final VersionConstraint version) => version,
        GitDependency() || PathDependency() => null,
      };
      if (version != null && version.toString() != pin.value) {
        throwToolExit(
          "${pin.key} should be pinned in $directory to version ${pin.value}, but isn't",
        );
      }
    }
  }

  void _checkHash(String pubspec, Directory directory) {
    final RegExpMatch? firstMatch = checksumRegex.firstMatch(pubspec);
    if (firstMatch == null) {
      throwToolExit('Pubspec in ${directory.path} does not contain a checksum.');
    }
    final String checksum = firstMatch[1]!;
    final String actualChecksum = _computeChecksum(pubspec);
    if (checksum != actualChecksum) {
      throwToolExit(
        'Pubspec in ${directory.path} has out of date dependencies. '
        'Please run "flutter update-packages --force-upgrade --update-hashes" to '
        'update them correctly. The hash ($checksum) does not match the '
        'expectation ($actualChecksum).',
      );
    }
  }

  void _writeHashesToPubspecs(List<Directory> packages) {
    globals.printStatus('Writing hashes to pubspecs...');
    for (final directory in packages) {
      globals.printTrace('Reading pubspec.yaml from ${directory.path}');
      final File pubspecFile = directory.childFile(_pubspecName);
      String pubspec = pubspecFile.readAsStringSync();
      final String actualChecksum = _computeChecksum(pubspec);
      final RegExpMatch? firstMatch = checksumRegex.firstMatch(pubspec);
      if (firstMatch != null) {
        pubspec = pubspec.replaceRange(
          firstMatch.start,
          firstMatch.end,
          '$kDependencyChecksum$actualChecksum',
        );
      } else {
        pubspec += '\n$kDependencyChecksum$actualChecksum';
      }
      pubspecFile.writeAsStringSync(pubspec);
    }
    globals.printStatus('All pubspecs are now up to date.');
  }

  String _computeChecksum(String pubspecString) {
    final pubspec = Pubspec.parse(pubspecString);
    return SplayTreeMap<String, Dependency>.from(<String, Dependency>{
          ...pubspec.dependencies.map(
            (String key, Dependency value) => MapEntry<String, Dependency>('dep:$key', value),
          ),
          ...pubspec.devDependencies.map(
            (String key, Dependency value) => MapEntry<String, Dependency>('dev_dep:$key', value),
          ),
          ...pubspec.dependencyOverrides.map(
            (String key, Dependency value) => MapEntry<String, Dependency>('dep_over:$key', value),
          ),
        }).entries
        .map((MapEntry<String, Dependency> entry) => '${entry.key}${entry.value}')
        .join()
        .hashCode
        .toRadixString(32);
  }

  /// This is the string we output next to each of our autogenerated transitive
  /// dependencies so that we can ignore them the next time we parse the
  /// pubspec.yaml file.
  static const kTransitiveMagicString =
      '# THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"';

  /// This is the string output before a checksum of the packages used.
  static const kDependencyChecksum = '# PUBSPEC CHECKSUM: ';
  final checksumRegex = RegExp('$kDependencyChecksum([a-zA-Z0-9]+)');
}

class ResolvedDependencies {
  ResolvedDependencies([Map<String, Map<String, String>>? data])
    : data = data ?? <String, Map<String, String>>{};

  static const _dependencies = 'dependencies';
  static const _devDependencies = 'dev_dependencies';
  final Map<String, Map<String, String>> data;

  List<PackageVersion> toVersions() {
    final versions = <PackageVersion>[];
    for (final dependencyType in <String>[_dependencies, _devDependencies]) {
      data[dependencyType] ??= <String, String>{};
      for (final MapEntry(key: package, value: version) in data[dependencyType]!.entries) {
        versions.add((package: package, version: version));
      }
    }
    return versions;
  }

  void forEach({
    required YamlEditor yamlEditor,
    required void Function(
      Map<String, String> dependencies,
      String depType,
      String packageName,
      Object? version,
    )
    func,
  }) {
    for (final dependencyType in <String>[_dependencies, _devDependencies]) {
      data[dependencyType] ??= <String, String>{};
      final Map<Object?, Object?> map =
          yamlEditor.parseAt(<String>[dependencyType], orElse: () => YamlMap()) as YamlMap;
      for (final MapEntry<Object?, Object?> dep in map.entries) {
        final packageName = dep.key! as String;
        final Object? restriction = dep.value;
        func(data[dependencyType]!, dependencyType, packageName, restriction);
      }
    }
  }

  static ResolvedDependencies mergeDeps(
    ResolvedDependencies oldDeps,
    ResolvedDependencies newDeps,
    List<PackageVersion> cherryPicks,
  ) {
    final mergedDeps = ResolvedDependencies(<String, Map<String, String>>{...newDeps.data});
    for (final MapEntry<String, Map<String, String>> entry in mergedDeps.data.entries) {
      final String dependencyType = entry.key;
      final Map<String, String>? oldData = oldDeps.data[dependencyType];
      for (final MapEntry<String, String> dep in entry.value.entries) {
        final String packageName = dep.key;
        final String newVersion = dep.value;
        final String? oldVersion =
            cherryPicks
                .where((PackageVersion pick) => pick.package == packageName)
                .map((PackageVersion pick) => pick.version)
                .firstOrNull ??
            oldData?[packageName];
        mergedDeps.data[dependencyType]![packageName] = oldVersion?.startsWith('^') ?? false
            ? _versionWithCaret(newVersion)
            : _versionWithoutCaret(newVersion);
      }
    }
    return mergedDeps;
  }
}

typedef PackageVersion = ({String package, String version});

/// How much dependencies should be relaxed when fetching new versions.
enum RelaxMode {
  /// Relax to an `any` dep, so major changes can be made.
  any,

  /// Relax to `^...`, so only minor changes can be made.
  caret,

  /// Do not relax, so keep the exact version.
  strict,
}

String _versionWithCaret(String version) => version.startsWith('^') ? version : '^$version';

String _versionWithoutCaret(String version) =>
    version.startsWith('^') ? version.substring(1) : version;