Add fixers to YAPF.
diff --git a/CHANGELOG b/CHANGELOG
index d0c0b80..59ae1fa 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,7 +2,12 @@
# All notable changes to this project will be documented in this file.
# This project adheres to [Semantic Versioning](http://semver.org/).
-## [0.29.1] UNRELEASED
+## [0.30.0] UNRELEASED
+### Added
+- Added a way to run lib2to3 fixers via the `--fixers` flag. The fixers live in
+ the `contrib/` directory, which isn't part of "official" yapf release.
+ Specifically, the code in `contrib/` may violate one of the core tenets of
+ yapf, and change the code itself and not just the whitespacing.
### Fixed
- Honor a disable directive at the end of a multiline comment.
- Don't require splitting before comments in a list when
diff --git a/README.rst b/README.rst
index 31f6d28..ece17ae 100644
--- a/README.rst
+++ b/README.rst
@@ -92,21 +92,22 @@
Options::
- usage: yapf [-h] [-v] [-d | -i] [-r | -l START-END] [-e PATTERN]
+ usage: yapf [-h] [-v] [-d | -i | -q] [-r | -l START-END] [-e PATTERN]
[--style STYLE] [--style-help] [--no-local-style] [-p]
- [-vv]
+ [--fixers {quotes}] [--force-quote-type {single,double}] [-vv]
[files [files ...]]
Formatter for Python code.
positional arguments:
- files
+ files reads from stdin when no files are specified.
optional arguments:
-h, --help show this help message and exit
-v, --version show version number and exit
-d, --diff print the diff for the fixed source
-i, --in-place make changes to files in place
+ -q, --quiet output nothing and set return value
-r, --recursive run recursively over directories
-l START-END, --lines START-END
range of lines to reformat, one-based
@@ -122,9 +123,12 @@
--style-help show style settings and exit; this output can be saved
to .style.yapf to make your settings permanent
--no-local-style don't search for local style definition
- -p, --parallel Run yapf in parallel when formatting multiple files.
+ -p, --parallel run yapf in parallel when formatting multiple files.
Requires concurrent.futures in Python 2.X
- -vv, --verbose Print out file names while processing
+ --fixers {quotes} comma-separated list of fixers to run on code
+ --force-quote-type {single,double}
+ type of quotes to use - ', ", or decide heuristically
+ -vv, --verbose print out file names while processing
------------
@@ -322,6 +326,48 @@
a == b
+lib2to3 Style Fixers
+====================
+
+YAPF allows you run `lib2to3 style fixers
+<http://python3porting.com/fixers.html>` before or after reformatting. The
+fixers live in the `contrib/fixers/` directory.
+
+Fixers modify the source code and not just whitespace, which is against YAPF's
+policy of changing only whitespace. Therefore, caution needs to be taken when
+using them so that the program's semantics don't change.
+
+------
+Quotes
+------
+
+The "quotes" fixer converts the quotes used to be consistent throughout the
+file. You can have YAPF heuristically choose the quote type (similar to how
+pylint chooses) or force a specific style.
+
+Heuristically choosing quote type:t
+
+.. code-block:: python
+
+ $ python -m yapf --fixers quotes < quotes.py
+ print("These quotes are double")
+
+ print("while these quotes are single")
+
+ print("We really want them to be the same!")
+
+Forcing single quote type:
+
+.. code-block:: python
+
+ $ python -m yapf --fixers quotes --force-quote-type single < quotes.py
+ print('These quotes are now single')
+
+ print('while these quotes are single')
+
+ print(We really want them to be the same!)
+
+
Knobs
=====
diff --git a/contrib/README.md b/contrib/README.md
new file mode 100644
index 0000000..c611f64
--- /dev/null
+++ b/contrib/README.md
@@ -0,0 +1,12 @@
+# YAPF contrib
+
+Any code in this directory is not officially supported, and may change or be
+removed at any time without notice.
+
+The `contrib/` directory contains external contributions to YAPF, which aren't
+part of its official code base. In particular, `lib2to3` "fixers" are added
+here. They aren't part of YAPF, because they modify non-whitespace in source
+code, which is against the YAPF philosophy.
+
+The code in this directory may not be tested on a regular basis, and thus
+should be used with caution.
diff --git a/contrib/fixers/__init__.py b/contrib/fixers/__init__.py
new file mode 100644
index 0000000..bafb0c9
--- /dev/null
+++ b/contrib/fixers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/contrib/fixers/fix_quotes.py b/contrib/fixers/fix_quotes.py
new file mode 100644
index 0000000..a181c4b
--- /dev/null
+++ b/contrib/fixers/fix_quotes.py
@@ -0,0 +1,94 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Fixer that enforces consistent use of quotes for strings.
+
+All strings should use the same quotes. The exception being when the opposite
+quote is used in the string itself --- i.e., '"' or "'". This also changes a
+docstring that uses ''' into one that uses double quotes, which is the required
+format.
+"""
+
+import re
+
+from contrib.fixers import line_conditional_fix
+
+
+class FixQuotes(line_conditional_fix.LineConditionalFix):
+ """Fixer for consistent use of quotes in strings."""
+
+ explicit = True # The user must ask for this fixer.
+
+ # Assorted regex patterns.
+ _delim_pattern = re.compile(r'^[uUbB]?[rR]?(?P<delim>"""|\'\'\'|"|\')')
+ _dbl_quote_pattern = re.compile(r'(^[uUbB]?[rR]?)"([^"]*)"$')
+ _sgl_quote_pattern = re.compile(r"(^[uUbB]?[rR]?)'([^']*)'$")
+ _multiline_string_quote_pattern = re.compile(
+ r"(?s)(^[uUbB]?[rR]?)'''(.*?)'''$")
+
+ PATTERN = """STRING"""
+
+ def __init__(self, options, log):
+ super(FixQuotes, self).__init__(options, log)
+
+ # The option force_quote_type permits forcing the quote style this fixer
+ # will enforce, regardless of what currently exists in the file.
+ # When it's 'none' or doesn't exist, there's no forcing and a heuristic
+ # is used to determine which quote style to use.
+ forced_quote_type = options.get('force_quote_type', 'none')
+ if forced_quote_type == 'single':
+ self._string_delim = "'"
+ elif forced_quote_type == 'double':
+ self._string_delim = '"'
+ else:
+ assert forced_quote_type == 'none'
+ self._string_delim = None
+
+ def transform(self, node, results):
+ if self.should_skip(node) or not node.parent:
+ return
+
+ first_delim = self._GetDelimiter(node.value)
+ if first_delim == "'''" and '"""' not in node.value[3:-3]:
+ # Always use """ for docstrings and multiline strings.
+ if node.value[-4] != '"':
+ new = node.clone()
+ new.parent = node.parent
+ new.value = re.sub(self._multiline_string_quote_pattern, r'\1"""\2"""',
+ node.value)
+ return new
+ return
+
+ if ((first_delim == '"' and "'" in node.value[1:-1]) or
+ (first_delim == "'" and '"' in node.value[1:-1])):
+ return
+
+ if self._string_delim is None:
+ self._string_delim = first_delim
+
+ elif first_delim != self._string_delim:
+ # The quote isn't consistent.
+ new = node.clone()
+ new.parent = node.parent
+ if self._string_delim == '"':
+ new.value = re.sub(self._sgl_quote_pattern, r'\1"\2"', node.value)
+ else:
+ new.value = re.sub(self._dbl_quote_pattern, r"\1'\2'", node.value)
+ return new
+
+ def _GetDelimiter(self, string):
+ """Return the delimiter for the given string repr, omitting prefix."""
+ match = re.search(self._delim_pattern, string)
+ if not match:
+ return None
+ return match.group('delim')
diff --git a/contrib/fixers/fixers_api.py b/contrib/fixers/fixers_api.py
new file mode 100644
index 0000000..742546d
--- /dev/null
+++ b/contrib/fixers/fixers_api.py
@@ -0,0 +1,79 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Entry point for refactoring via the lib2to3 fixer."""
+
+from lib2to3 import refactor as lib2to3_refactor
+from lib2to3.pgen2 import parse as pgen2_parse
+from lib2to3.pgen2 import tokenize as pgen2_tokenize
+
+# Our path in the source tree.
+MODULE_NAME_PREFIX = 'contrib.fixers.fix_'
+
+# A list of available fixers.
+AVAILABLE_FIXERS = ['quotes']
+
+
+def ValidateCommandLineArguments(args):
+ if args.fixers:
+ for fixer in args.fixers:
+ if fixer not in AVAILABLE_FIXERS:
+ return 'invalid fixer specified: ' + fixer
+
+ if ((not args.fixers or 'quotes' not in args.fixers) and
+ args.force_quote_type != 'none'):
+ return 'cannot use --force-quote-type without --fixers=quotes'
+
+ return None
+
+
+def Pre2to3FixerRun(original_source, options):
+ """2to3 fixers to run before reformatting the file."""
+ if options and options['fixers']:
+ return _Run2to3Fixers(original_source, options=options)
+ return original_source
+
+
+def Post2to3FixerRun(original_source, options):
+ """2to3 fixers to run after reformatting the file."""
+ if options and options['fixers']:
+ return _Run2to3Fixers(original_source, options)
+ return original_source
+
+
+def _Run2to3Fixers(source, options):
+ """Use lib2to3 to reformat the source.
+
+ Args:
+ source: (unicode) The source to reformat.
+ options: dictionary of options to pass to lib2to3_refactor.RefactoringTool
+
+ Returns:
+ Reformatted source code.
+ """
+ fixer_names = [MODULE_NAME_PREFIX + fixer for fixer in options['fixers']]
+ options['print_function'] = True
+ try:
+ try:
+ tool = lib2to3_refactor.RefactoringTool(
+ fixer_names=fixer_names, explicit=fixer_names, options=options)
+ return '{}'.format(tool.refactor_string(source, name=''))
+ except pgen2_parse.ParseError:
+ options['print_function'] = False
+ tool = lib2to3_refactor.RefactoringTool(
+ fixer_names=fixer_names, explicit=fixer_names, options=options)
+ return '{}'.format(tool.refactor_string(source, name=''))
+ except (pgen2_tokenize.TokenError, pgen2_parse.ParseError, SyntaxError,
+ UnicodeDecodeError, UnicodeEncodeError) as err:
+ logging.error(err)
+ raise
diff --git a/contrib/fixers/line_conditional_fix.py b/contrib/fixers/line_conditional_fix.py
new file mode 100644
index 0000000..f414a22
--- /dev/null
+++ b/contrib/fixers/line_conditional_fix.py
@@ -0,0 +1,102 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Fixer base for skipping nodes not within a line range."""
+
+import collections
+import re
+
+from lib2to3 import fixer_base
+from lib2to3 import pytree
+
+
+class LineConditionalFix(fixer_base.BaseFix):
+ """Base class for fixers which only want to execute on certain lines."""
+
+ def __init__(self, options, log):
+ super(LineConditionalFix, self).__init__(options, log)
+ # Note that for both _skip_nodes and _line_elems, we have to store the
+ # pytree node's "id" in the set, because pytree nodes aren't hashable.
+ self._skip_nodes = set()
+ self._line_elems = collections.defaultdict(set)
+
+ def start_tree(self, tree, filename):
+ """Record the initial position of nodes and determine which to skip."""
+ super(LineConditionalFix, self).start_tree(tree, filename)
+
+ disabled_region = False
+ for node in tree.pre_order():
+ if _IsYapfEnableNode(node):
+ disabled_region = False
+
+ if disabled_region:
+ self._skip_nodes.add(id(node))
+
+ start_leaf, stop_leaf = _GetLeaf(node, 0), _GetLeaf(node, -1)
+ if start_leaf is None or stop_leaf is None:
+ self._skip_nodes.add(id(node)) # cannot determine line numbers, so skip
+ elif _OutsideOfLineRanges(start_leaf.lineno, stop_leaf.lineno,
+ self.options.get('lines')):
+ self._skip_nodes.add(id(node))
+
+ if _IsYapfDisableNode(node):
+ num_newlines = node.prefix.count('\n')
+ # A tailing comment that disables yapf acts on the previous line.
+ # lib2to3 helpfully adds that comment to the first node of the next
+ # line. Therefore, when we see a trailing comment, we need to go back to
+ # the previous line and add those elements to the "self._skip_nodes"
+ # set.
+ if (not node.prefix.startswith('\n') and
+ self._line_elems[node.get_lineno() - num_newlines]):
+ for elem in self._line_elems[node.get_lineno() - num_newlines]:
+ self._skip_nodes.add(elem)
+ else:
+ disabled_region = True
+
+ self._line_elems[node.get_lineno()].add(id(node))
+
+ def should_skip(self, node):
+ """Returns true if this node isn't in the specified line range."""
+ return id(node) in self._skip_nodes
+
+
+_DISABLE_PATTERN = r'^\s*#.+yapf:\s*disable\b'
+_ENABLE_PATTERN = r'^\s*#.+yapf:\s*enable\b'
+
+
+def _OutsideOfLineRanges(start_lineno, stop_lineno, lines):
+ if not lines:
+ return False
+
+ for start, stop in reversed(lines):
+ if start_lineno >= start and stop_lineno <= stop:
+ return False
+
+ return True
+
+
+def _IsYapfDisableNode(node):
+ return re.search(_DISABLE_PATTERN, node.prefix.strip(), re.I | re.M)
+
+
+def _IsYapfEnableNode(node):
+ return re.search(_ENABLE_PATTERN, node.prefix.strip(), re.I | re.M)
+
+
+def _GetLeaf(node, index):
+ """A helper method to get the left-most or right-most child of a node."""
+ while not isinstance(node, pytree.Leaf):
+ if not node.children:
+ return None
+ node = node.children[index]
+ return node
diff --git a/yapf/__init__.py b/yapf/__init__.py
index 41b2a5d..b40f968 100644
--- a/yapf/__init__.py
+++ b/yapf/__init__.py
@@ -32,6 +32,8 @@
import os
import sys
+from contrib.fixers import fixers_api
+
from lib2to3.pgen2 import tokenize
from yapf.yapflib import errors
@@ -57,7 +59,8 @@
Raises:
YapfError: if none of the supplied files were Python files.
"""
- parser = argparse.ArgumentParser(description='Formatter for Python code.')
+ parser = argparse.ArgumentParser(
+ prog='yapf', description='Formatter for Python code.')
parser.add_argument(
'-v',
'--version',
@@ -121,19 +124,35 @@
'--no-local-style',
action='store_true',
help="don't search for local style definition")
- parser.add_argument('--verify', action='store_true', help=argparse.SUPPRESS)
parser.add_argument(
'-p',
'--parallel',
action='store_true',
help=('run yapf in parallel when formatting multiple files. Requires '
'concurrent.futures in Python 2.X'))
+
+ parser.add_argument(
+ '--fixers',
+ action='store',
+ default=None,
+ metavar='{' + ', '.join(fixers_api.AVAILABLE_FIXERS) + '}',
+ type=lambda x: [s.strip().lower() for s in x.split(',')],
+ help='comma-separated list of fixers to run on code')
+ parser.add_argument(
+ '--force-quote-type',
+ action='store',
+ default='none',
+ choices=['single', 'double'],
+ help='type of quotes to use - \', ", or decide heuristically')
+
parser.add_argument(
'-vv',
'--verbose',
action='store_true',
help='print out file names while processing')
+ parser.add_argument('--verify', action='store_true', help=argparse.SUPPRESS)
+
parser.add_argument(
'files', nargs='*', help='reads from stdin when no files are specified.')
args = parser.parse_args(argv[1:])
@@ -151,7 +170,19 @@
if args.lines and len(args.files) > 1:
parser.error('cannot use -l/--lines with more than one file')
+ err_msg = fixers_api.ValidateCommandLineArguments(args)
+ if err_msg:
+ parser.error(err_msg)
+ return 1
+
lines = _GetLines(args.lines) if args.lines is not None else None
+
+ options_for_fixers = {
+ 'fixers': args.fixers,
+ 'force_quote_type': args.force_quote_type,
+ 'lines': lines,
+ }
+
if not args.files:
# No arguments specified. Read code from stdin.
if args.in_place or args.diff:
@@ -184,6 +215,7 @@
py3compat.unicode('\n'.join(source) + '\n'),
filename='<stdin>',
style_config=style_config,
+ options=options_for_fixers,
lines=lines,
verify=args.verify)
except tokenize.TokenError as e:
@@ -211,6 +243,7 @@
print_diff=args.diff,
verify=args.verify,
parallel=args.parallel,
+ options=options_for_fixers,
quiet=args.quiet,
verbose=args.verbose)
return 1 if changed and (args.diff or args.quiet) else 0
@@ -239,9 +272,10 @@
no_local_style=False,
in_place=False,
print_diff=False,
- verify=False,
parallel=False,
+ options=None,
quiet=False,
+ verify=False,
verbose=False):
"""Format a list of files.
@@ -257,9 +291,10 @@
in_place: (bool) Modify the files in place.
print_diff: (bool) Instead of returning the reformatted source, return a
diff that turns the formatted source into reformatter source.
- verify: (bool) True if reformatted code should be verified for syntax.
parallel: (bool) True if should format multiple files in parallel.
+ options: (dict) options for running fixers.
quiet: (bool) True if should output nothing.
+ verify: (bool) True if reformatted code should be verified for syntax.
verbose: (bool) True if should print out filenames while processing.
Returns:
@@ -281,7 +316,8 @@
else:
for filename in filenames:
changed |= _FormatFile(filename, lines, style_config, no_local_style,
- in_place, print_diff, verify, quiet, verbose)
+ in_place, print_diff, options, verify, quiet,
+ verbose)
return changed
@@ -291,6 +327,7 @@
no_local_style=False,
in_place=False,
print_diff=False,
+ options=None,
verify=False,
quiet=False,
verbose=False):
@@ -309,6 +346,7 @@
style_config=style_config,
lines=lines,
print_diff=print_diff,
+ options=options,
verify=verify,
logger=logging.warning)
except tokenize.TokenError as e:
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
index dde1df9..c56aced 100644
--- a/yapf/yapflib/yapf_api.py
+++ b/yapf/yapflib/yapf_api.py
@@ -36,6 +36,8 @@
import re
import sys
+from contrib.fixers import fixers_api
+
from lib2to3.pgen2 import parse
from yapf.yapflib import blank_line_calculator
@@ -56,8 +58,9 @@
style_config=None,
lines=None,
print_diff=False,
- verify=False,
+ options=None,
in_place=False,
+ verify=False,
logger=None):
"""Format a single Python file and return the formatted code.
@@ -72,8 +75,9 @@
than a whole file.
print_diff: (bool) Instead of returning the reformatted source, return a
diff that turns the formatted source into reformatter source.
- verify: (bool) True if reformatted code should be verified for syntax.
+ options: (dict) options for running fixers.
in_place: (bool) If True, write the reformatted code back to the file.
+ verify: (bool) True if reformatted code should be verified for syntax.
logger: (io streamer) A stream to output logging.
Returns:
@@ -97,6 +101,7 @@
filename=filename,
lines=lines,
print_diff=print_diff,
+ options=options,
verify=verify)
if reformatted_source.rstrip('\n'):
lines = reformatted_source.rstrip('\n').split('\n')
@@ -115,6 +120,7 @@
style_config=None,
lines=None,
print_diff=False,
+ options=None,
verify=False):
"""Format a string of Python code.
@@ -132,6 +138,7 @@
than a whole file.
print_diff: (bool) Instead of returning the reformatted source, return a
diff that turns the formatted source into reformatter source.
+ options: (dict) options for running fixers.
verify: (bool) True if reformatted code should be verified for syntax.
Returns:
@@ -143,6 +150,8 @@
if not unformatted_source.endswith('\n'):
unformatted_source += '\n'
+ unformatted_source = fixers_api.Pre2to3FixerRun(unformatted_source, options)
+
try:
tree = pytree_utils.ParseCodeToTree(unformatted_source)
except parse.ParseError as e:
