Add process runner utility, update example
diff --git a/CHANGELOG.md b/CHANGELOG.md index 57e64a5..7df0649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -1,5 +1,11 @@ # Change Log for `process_runner` +## 4.1.0 + +* Adds a pub-installable command line utility, based on the example code, to run a tasks queue of commands from a command line. See [README.md](README.md) for more details. +* Example code is updated. +* Now throws a `ProcessRunnerException` if a job fails and `failOk` on that job is `false`. + ## 4.0.1 * Add startMode to allow passing of a ProcessStartMode to
diff --git a/README.md b/README.md index 4b67534..f2c26f8 100644 --- a/README.md +++ b/README.md
@@ -17,7 +17,7 @@ manages running them with a set number of active [`WorkerJob`s], and manages the collection of their stdout, stderr, and interleaved stdout and stderr output. -See the [example](example/main.dart) and [`process_runner` library] docs for +See the [example](example/main.dart) and [`process_runner` library docs] for more information on how to use it, but the basic usage for is: ```dart @@ -71,11 +71,79 @@ } ``` +## `process_runner` utility + +The example can also be installed and run as a useful command-line utility. You can install it using: + +```shell +dart pub global activate process_runner +``` + +And you can run it with: + +```shell +dart pub global run process_runner +``` + +The above steps will work on any Dart-supported platform. + +Of course, you can also just compile the example into a native executable and move it to a directory in your PATH: + +```shell +dart compile exe bin/process_runner.dart -o process_runner +mv process_runner /some/bin/dir/in/your/path +``` + +The usage for the utility is as follows: + +``` +process_runner [--help] [--quiet] [--report] [--stdout] [--stderr] + [--run-in-shell] [--working-directory=<working directory>] + [--jobs=<num_worker_jobs>] [--command="command" ...] + [--source=<file|"-"> ...]: +-h, --help Print help for process_runner. +-q, --quiet Silences the stderr and stdout output of the + commands. This is a shorthand for "--no-stdout + --no-stderr". +-r, --report Print progress on the jobs to stderr while running. + --[no-]stdout Prints the stdout output of the commands to stdout in + the order they complete. Will not interleave lines + from separate processes. Has no effect if --quiet is + specified. + (defaults to on) + --[no-]stderr Prints the stderr output of the commands to stderr in + the order they complete. Will not interleave lines + from separate processes. Has no effect if --quiet is + specified + (defaults to on) + --run-in-shell Run the commands in a subshell. + --[no-]fail-ok If set, allows continuing execution of the remaining + commands even if one fails to execute. If not set, + ("--no-fail-ok") then process will just exit with a + non-zero code at completion if there were any jobs + that failed. +-j, --jobs Specify the number of worker jobs to run + simultaneously. Defaults to the number of processor + cores on the machine. + --working-directory Specify the working directory to run in. + (defaults to ".") +-c, --command Specify a command to add to the commands to be run. + Commands specified with this option run before those + specified with --source. Be sure to quote arguments + to --command properly on the command line. +-s, --source Specify the name of a file to read commands from, one + per line, as they would appear on the command line, + with spaces escaped or quoted. Specify "--source -" + to read from stdin. More than one --source argument + may be specified, and they will be concatenated in + the order specified. The stdin ("--source -") + argument may only be specified once. +``` [`ProcessManager`]: https://github.com/google/process.dart/blob/master/lib/src/interface/process_manager.dart#L21 [`process`]: https://pub.dev/packages/process [`process_runner`]: https://pub.dev/packages/process_runner [`ProcessRunner`]: https://pub.dev/documentation/process_runner/latest/process_runner/ProcessRunner-class.html [`ProcessPool`]: https://pub.dev/documentation/process_runner/latest/process_runner/ProcessPool-class.html -[`process_runner` library]: https://pub.dev/documentation/process_runner/latest/process_runner/process_runner-library.html +[`process_runner` library docs]: https://pub.dev/documentation/process_runner/latest/process_runner/process_runner-library.html [`WorkerJob`s]: https://pub.dev/documentation/process_runner/latest/process_runner/WorkerJob-class.html
diff --git a/bin/process_runner.dart b/bin/process_runner.dart new file mode 100644 index 0000000..fc8c6bc --- /dev/null +++ b/bin/process_runner.dart
@@ -0,0 +1,18 @@ +// Copyright 2020 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This utility sends a bunch of jobs to a ProcessPool for processing. +// +// It can speed up processing of a bunch of single-threaded CPU-intensive +// commands by a multiple of the number of processor cores you have (modulo being +// disk/network bound, of course). +// +// You can install this command with "dart pub global activate process_runner", +// and you can run it with "dart pub global run process_runner". + +import '../example/main.dart' as runner_main; + +Future<void> main(List<String> args) async { + return runner_main.main(args); +}
diff --git a/example/main.dart b/example/main.dart index 294c3c8..ebef3e0 100644 --- a/example/main.dart +++ b/example/main.dart
@@ -6,7 +6,7 @@ // // This example program is actually pretty useful even if you don't use // process_runner for your Dart project. It can speed up processing of a bunch -// of single-threaded CPU-intensive commands by a multple of the number of +// of single-threaded CPU-intensive commands by a multiple of the number of // processor cores you have (modulo being disk/network bound, of course). import 'dart:io'; @@ -14,6 +14,27 @@ import 'package:args/args.dart'; import 'package:process_runner/process_runner.dart'; +const String _kHelpFlag = 'help'; +const String _kQuietFlag = 'quiet'; +const String _kReportFlag = 'report'; +const String _kPrintStdoutFlag = 'stdout'; +const String _kPrintStderrFlag = 'stderr'; +const String _kRunInShellFlag = 'run-in-shell'; +const String _kAllowFailureFlag = 'fail-ok'; +const List<String> _kFlags = <String>[ + _kHelpFlag, + _kQuietFlag, + _kReportFlag, + _kPrintStdoutFlag, + _kPrintStderrFlag, + _kRunInShellFlag, +]; +const String _kJobsOption = 'jobs'; +const String _kWorkingDirectoryOption = 'working-directory'; +const String _kCommandOption = 'command'; +const String _kSourceOption = 'source'; +const String _kAppName = 'process_runner'; + // This only works for escaped spaces and things in double or single quotes. // This is just an example, modify to meet your own requirements. List<String> splitIntoArgs(String args) { @@ -93,64 +114,169 @@ } } -Future<void> main(List<String> args) async { - final ArgParser parser = ArgParser(); - parser.addFlag('help', help: 'Print help.'); - parser.addFlag('report', help: 'Print progress on the jobs while running.', defaultsTo: false); - parser.addFlag('run-in-shell', help: 'Run the commands in a subshell.', defaultsTo: false); - parser.addOption('workers', - abbr: 'w', - help: 'Specify the number of workers jobs to run simultanously. Defaults ' - 'to the number of processors on the machine.'); - parser.addOption('workingDirectory', - abbr: 'd', help: 'Specify the working directory to run on', defaultsTo: '.'); - parser.addMultiOption('cmd', - abbr: 'c', - help: 'Specify a command to add to the commands to be run. Entire ' - 'command must be quoted by the shell. Commands specified with this ' - 'option run before those specified with --cmdFile'); - parser.addOption('file', - abbr: 'f', - help: 'Specify the name of a file to read commands from, one per line, as ' - 'they would appear on the command line, with spaces escaped or ' - 'quoted. Specify "-" to read from stdin.', - defaultsTo: '-'); - final ArgResults options = parser.parse(args); +// Print reports to stderr, to avoid polluting any stdout from the jobs. +void stderrPrintReport( + int total, + int completed, + int inProgress, + int pending, + int failed, +) { + stderr.write(ProcessPool.defaultReportToString(total, completed, inProgress, pending, failed)); +} - if (options['help'] as bool) { - print('main.dart [flags]'); - print(parser.usage); - exit(0); +Future<void> main(List<String> args) async { + final ArgParser parser = ArgParser(usageLineLength: 80); + parser.addFlag( + _kHelpFlag, + abbr: 'h', + defaultsTo: false, + negatable: false, + help: 'Print help for $_kAppName.', + ); + parser.addFlag( + _kQuietFlag, + abbr: 'q', + defaultsTo: false, + negatable: false, + help: 'Silences the stderr and stdout output of the commands. This ' + 'is a shorthand for "--no-$_kPrintStdoutFlag --no-$_kPrintStderrFlag".', + ); + parser.addFlag( + _kReportFlag, + abbr: 'r', + defaultsTo: false, + negatable: false, + help: 'Print progress on the jobs to stderr while running.', + ); + parser.addFlag( + _kPrintStdoutFlag, + defaultsTo: true, + help: 'Prints the stdout output of the commands to stdout in the order ' + 'they complete. Will not interleave lines from separate processes. Has no ' + 'effect if --$_kQuietFlag is specified.', + ); + parser.addFlag( + _kPrintStderrFlag, + defaultsTo: true, + help: 'Prints the stderr output of the commands to stderr in the order ' + 'they complete. Will not interleave lines from separate processes. Has no ' + 'effect if --$_kQuietFlag is specified', + ); + parser.addFlag( + _kRunInShellFlag, + defaultsTo: false, + negatable: false, + help: 'Run the commands in a subshell.', + ); + parser.addFlag( + _kAllowFailureFlag, + defaultsTo: false, + help: 'If set, allows continuing execution of the remaining commands even if ' + 'one fails to execute. If not set, ("--no-$_kAllowFailureFlag") then ' + 'process will just exit with a non-zero code at completion if there were ' + 'any jobs that failed.', + ); + parser.addOption( + _kJobsOption, + abbr: 'j', + help: 'Specify the number of worker jobs to run simultaneously. Defaults ' + 'to the number of processor cores on the machine (which is ' + '${Platform.numberOfProcessors} on this machine).', + ); + parser.addOption( + _kWorkingDirectoryOption, + defaultsTo: '.', + help: 'Specify the working directory to run in.', + ); + parser.addMultiOption( + _kCommandOption, + abbr: 'c', + help: 'Specify a command to add to the commands to be run. Commands ' + 'specified with this option run before those specified with ' + '--$_kSourceOption. Be sure to quote arguments to --$_kCommandOption ' + 'properly on the command line.', + ); + parser.addMultiOption( + _kSourceOption, + abbr: 's', + defaultsTo: <String>[], + help: 'Specify the name of a file to read commands from, one per line, as ' + 'they would appear on the command line, with spaces escaped or ' + 'quoted. Specify "--$_kSourceOption -" to read from stdin. More than ' + 'one --$_kSourceOption argument may be specified, and they will be ' + 'concatenated in the order specified. The stdin ("--$_kSourceOption -") ' + 'argument may only be specified once.', + ); + + late ArgResults options; + try { + options = parser.parse(args); + } on FormatException catch (e) { + stderr.writeln('Argument Error: ${e.message}'); + stderr.writeln(parser.usage); + exitCode = 1; + return; } - // Collect the commands to be run from the command file. - final String? commandFile = options['file'] as String?; - List<String> fileCommands = <String>[]; - if (commandFile != null) { - // Read from stdin if the --file option is set to '-'. - if (commandFile == '-') { - String? line = stdin.readLineSync(); - while (line != null) { - fileCommands.add(line); - line = stdin.readLineSync(); + if (options[_kHelpFlag] as bool) { + print( + '$_kAppName [--${_kFlags.join('] [--')}] ' + '[--$_kWorkingDirectoryOption=<working directory>] ' + '[--$_kJobsOption=<num_worker_jobs>] ' + '[--$_kCommandOption="command" ...] ' + '[--$_kSourceOption=<file|"-"> ...]:', + ); + + print(parser.usage); + exitCode = 0; + return; + } + + final bool quiet = options[_kQuietFlag]! as bool; + final bool printStderr = !quiet && options[_kPrintStderrFlag]! as bool; + final bool printStdout = !quiet && options[_kPrintStdoutFlag]! as bool; + final bool printReport = options[_kReportFlag]! as bool; + final bool runInShell = options[_kRunInShellFlag]! as bool; + final bool failOk = options[_kAllowFailureFlag]! as bool; + + // Collect the commands to be run from the command file(s). + final List<String>? commandFiles = options[_kSourceOption] as List<String>?; + final List<String> fileCommands = <String>[]; + if (commandFiles != null) { + bool sawStdinAlready = false; + for (final String commandFile in commandFiles) { + // Read from stdin if the --file option is set to '-'. + if (commandFile == '-') { + if (sawStdinAlready) { + stderr.writeln('ERROR: The stdin can only be specified once with "--$_kSourceOption -"'); + exitCode = 1; + return; + } + sawStdinAlready = true; + String? line = stdin.readLineSync(); + while (line != null) { + fileCommands.add(line); + line = stdin.readLineSync(); + } + } else { + // Read the commands from a file. + final File cmdFile = File(commandFile); + if (!cmdFile.existsSync()) { + print('''Command file "$commandFile" doesn't exist.'''); + exit(1); + } + fileCommands.addAll(cmdFile.readAsLinesSync()); } - } else { - // Read the commands from a file. - final File cmdFile = File(commandFile); - if (!cmdFile.existsSync()) { - print('Command file "$commandFile" doesn\'t exist.'); - exit(1); - } - fileCommands = cmdFile.readAsLinesSync(); } } // Collect all the commands, both from the input file, and from the command - // line. The command line commands come first (although they could all be - // executed simultaneously, depending on the number of workers, and number of - // commands). + // line. The command line commands come first (although they could all + // potentially be executed simultaneously, depending on the number of workers, + // and number of commands). final List<String> commands = <String>[ - if (options['cmd'] != null) ...options['cmd']! as List<String>, + if (options[_kCommandOption] != null) ...options[_kCommandOption]! as List<String>, ...fileCommands, ]; @@ -160,26 +286,42 @@ // If the numWorkers is set to null, then the ProcessPool will automatically // select the number of processes based on how many CPU cores the machine has. - final int? numWorkers = int.tryParse(options['workers'] as String? ?? ''); - final bool printReport = options['report']! as bool? ?? false; - final Directory workingDirectory = Directory((options['workingDirectory'] as String?) ?? '.'); - final bool runInShell = options['run-in-shell'] as bool? ?? false; + final int? numWorkers = int.tryParse(options[_kJobsOption] as String? ?? ''); + final Directory workingDirectory = + Directory((options[_kWorkingDirectoryOption] as String?) ?? '.'); final ProcessPool pool = ProcessPool( numWorkers: numWorkers, - printReport: printReport ? ProcessPool.defaultPrintReport : null, + printReport: printReport ? stderrPrintReport : null, ); final List<WorkerJob> jobs = splitCommands.map<WorkerJob>((List<String> command) { return WorkerJob( command, workingDirectory: workingDirectory, runInShell: runInShell, + failOk: failOk, ); }).toList(); - await for (final WorkerJob done in pool.startWorkers(jobs)) { - if (printReport) { - print('\nFinished job ${done.name}'); + try { + await for (final WorkerJob done in pool.startWorkers(jobs)) { + if (printReport) { + stderr.writeln('\nFinished job ${done.name}'); + } + if (printStdout) { + stdout.write(done.result.stdout); + } + if (printStderr) { + stderr.write(done.result.stderr); + } } - stdout.write(done.result.stdout); + } on ProcessRunnerException catch (e) { + if (!failOk) { + stderr.writeln('$_kAppName execution failed: $e'); + exitCode = e.exitCode; + return; + } } + + // Return non-zero exit code if there were jobs that failed. + exitCode = pool.failedJobs != 0 ? 1 : 0; }
diff --git a/lib/src/process_pool.dart b/lib/src/process_pool.dart index cf5c015..ed2c04c 100644 --- a/lib/src/process_pool.dart +++ b/lib/src/process_pool.dart
@@ -56,16 +56,26 @@ /// Whether or not this command should print it's stdout when it runs. final bool printOutput; - /// Whether or not failure of this job should print a message to stderr or - /// not. + /// Whether or not failure of this job should throw an exception. + /// + /// If `failOk` is false, and this job fails (returns a non-zero exit code, or + /// otherwise fails to start), then a [ProcessRunnerException] will be thrown + /// containing the details. /// /// Defaults to true, since the [result] will contain the exit code. final bool failOk; - /// If set to true, the process will run in a shell. + /// If set to true, the process will run be spawned through a system shell. /// /// Running in a shell is generally not recommended, as it provides worse - /// performance, and some security risk, but is sometimes necessary. + /// performance, and some security risk, but is sometimes necessary for + /// accessing the shell environment. Shell command line expansion and + /// interpolation is not performed on the commands, but you can execute shell + /// builtins. Use the shell builtin "eval" (on Unix systems) if you want to + /// execute shell commands with expansion. + /// + /// On Linux and OS X, `/bin/sh` is used, while on Windows, + /// `%WINDIR%\system32\cmd.exe` is used. /// /// Defaults to false. final bool runInShell; @@ -168,8 +178,7 @@ totalJobs, _completedJobs.length, _inProgressJobs, _pendingJobs.length, _failedJobs.length); } - /// The default report printing function, if one is not supplied. - static void defaultPrintReport( + static String defaultReportToString( int total, int completed, int inProgress, @@ -183,9 +192,18 @@ final String inProgressStr = inProgress.toString().padLeft(2); final String pendingStr = pending.toString().padLeft(3); final String failedStr = failed.toString().padLeft(3); + return 'Jobs: $percent% done, $completedStr/$totalStr completed, $inProgressStr in progress, $pendingStr pending, $failedStr failed. \r'; + } - stdout.write( - 'Jobs: $percent% done, $completedStr/$totalStr completed, $inProgressStr in progress, $pendingStr pending, $failedStr failed. \r'); + /// The default report printing function, if one is not supplied. + static void defaultPrintReport( + int total, + int completed, + int inProgress, + int pending, + int failed, + ) { + stdout.write(defaultReportToString(total, completed, inProgress, pending, failed)); } Future<WorkerJob> _performJob(WorkerJob job) async { @@ -204,12 +222,12 @@ ); _completedJobs.add(job); } on ProcessRunnerException catch (e) { - if (!job.failOk) { - stderr.writeln('\nJob $job failed: $e'); - } job.result = e.result ?? ProcessRunnerResult.failed; job.exception = e; _failedJobs.add(job); + if (!job.failOk) { + rethrow; + } } finally { _inProgressJobs--; _printReportIfNeeded();
diff --git a/pubspec.yaml b/pubspec.yaml index 6115083..2a44609 100644 --- a/pubspec.yaml +++ b/pubspec.yaml
@@ -3,7 +3,7 @@ # found in the LICENSE file. name: process_runner -version: 4.0.1 +version: 4.1.0 description: A process invocation astraction for Dart that manages a multiprocess queue. homepage: https://github.com/google/process_runner @@ -20,3 +20,6 @@ environment: sdk: '>=2.12.0 <3.0.0' + +binaries: + process_runner:
diff --git a/test/src/process_pool_test.dart b/test/src/process_pool_test.dart index 8aed900..8d2d953 100644 --- a/test/src/process_pool_test.dart +++ b/test/src/process_pool_test.dart
@@ -72,7 +72,22 @@ expect(completed.first.result.stderr, equals('stderr1')); expect(completed.first.result.output, equals('output1stderr1')); }); - test('Commands the throw exceptions report results', () async { + test('failed tests throw when failOk is false', () async { + final Map<FakeInvocationRecord, List<ProcessResult>> calls = + <FakeInvocationRecord, List<ProcessResult>>{ + FakeInvocationRecord(<String>['command', 'arg1', 'arg2'], testPath): <ProcessResult>[ + ProcessResult(0, -1, 'output1', 'stderr1'), + ], + }; + fakeProcessManager.fakeResults = calls; + final List<WorkerJob> jobs = <WorkerJob>[ + WorkerJob(<String>['command', 'arg1', 'arg2'], name: 'job 1', failOk: false), + ]; + expect(() async { + await processPool.runToCompletion(jobs); + }, throwsException); + }); + test('Commands that throw exceptions report results', () async { fakeProcessManager = FakeProcessManager((String value) {}, commandsThrow: true); processRunner = ProcessRunner(processManager: fakeProcessManager); processPool = ProcessPool(processRunner: processRunner, printReport: null);