diff options
author | Yuexi Ma <yuexima@google.com> | 2018-07-26 11:02:32 -0700 |
---|---|---|
committer | android-build-merger <android-build-merger@google.com> | 2018-07-26 11:02:32 -0700 |
commit | 916bf2ba98abf7bb69bcc949603fa7f5f18af412 (patch) | |
tree | e0b807c3a311f5f549995eabd9f89e0e3f558eae | |
parent | 3c3bef0be1c91e1637f9a6ad664fcd511b562759 (diff) | |
parent | 91ce87b36eb4a923d5d9c2982905ecd28295e2b4 (diff) | |
download | yapf-916bf2ba98abf7bb69bcc949603fa7f5f18af412.tar.gz |
Initial merge from 'aosp/upstream-master' am: ef2f9b5c46 am: 07f9bff504 am: aa329ec328
am: 91ce87b36e
Change-Id: Ib8995bd74388036bacbdd4189b8c6f3cb084053f
72 files changed, 19936 insertions, 0 deletions
diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..3c3efd7 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +# Verifier is used for testing only. +omit = + */__main__.py + */verifier.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0736228 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,14 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# 2 space indentation +[*.py] +indent_style = space +indent_size = 2 @@ -0,0 +1,8 @@ +[flake8] +ignore = + # indentation is not a multiple of four, + E111,E114, + # visually indented line with same indent as next logical line, + E129 + +max-line-length=80 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3fd6b2b --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +#==============================================================================# +# This file specifies intentionally untracked files that git should ignore. +# See: http://www.kernel.org/pub/software/scm/git/docs/gitignore.html +# +# This file is intentionally different from the output of `git svn show-ignore`, +# as most of those are useless. +#==============================================================================# + +#==============================================================================# +# File extensions to be ignored anywhere in the tree. +#==============================================================================# +# Temp files created by most text editors. +*~ +# Merge files created by git. +*.orig +# Byte compiled python modules. +*.pyc +# vim swap files +.*.sw? +.sw? +# OS X specific files. +.DS_store + +#==============================================================================# +# Files to ignore +#==============================================================================# +/.coverage + +# Directories to ignore (do not add trailing '/'s, they skip symlinks). +#==============================================================================# +/build +/dist +/.tox +/yapf.egg-info diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000..823a973 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +# YAPF uses the chromium style +based_on_style = chromium diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1d90518 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,28 @@ +language: python + +python: + - 2.7 + - 3.4 + - 3.5 + - 3.6 + - nightly + +matrix: + allow_failures: + - python: nightly + include: + - python: 2.7 + env: SCA=true + - python: 3.5 + env: SCA=true + +install: + - if [ -z "$SCA" ]; then pip install --quiet coveralls; else echo skip; fi + - if [ -n "$SCA" ]; then python setup.py develop; else echo skip; fi + +script: + - if [ -n "$SCA" ]; then yapf --diff --recursive . || exit; else echo skip; fi + - if [ -z "$SCA" ]; then nosetests --with-coverage --cover-package=yapf; else echo skip; fi + +after_success: + - coveralls @@ -0,0 +1,4 @@ +" Force indentation styles for this directory +autocmd FileType python set shiftwidth=2 +autocmd FileType python set tabstop=2 +autocmd FileType python set softtabstop=2 @@ -0,0 +1,9 @@ +# This is the official list of YAPF authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization <email address> +# The email address is not required for organizations. + +Google Inc. diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..4693f03 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,565 @@ +# Change Log +# All notable changes to this project will be documented in this file. +# This project adheres to [Semantic Versioning](http://semver.org/). + +## [0.23.0] UNRELEASED +### Added +- `DISABLE_ENDING_COMMA_HEURISTIC` is a new knob to disable the heuristic which + splits a list onto separate lines if the list is comma-terminated. +### Fixed +- There's no need to increase N_TOKENS. In fact, it causes other things which + use lib2to3 to fail if called from YAPF. +- Change the exception message instead of creating a new one that's just a + clone. + +## [0.22.0] 2018-05-15 +### Added +- The `BLANK_LINE_BEFORE_MODULE_DOCSTRING` knob adds a blank line before a + module's docstring. +- The `SPLIT_ALL_COMMA_SEPARATED_VALUES` knob causes all lists, tuples, dicts + function defs, etc... to split on all values, instead of maximizing the + number of elements on each line, when not able to fit on a single line. +### Changed +- Improve the heuristic we use to determine when to split at the start of a + function call. First check whether or not all elements can fit in the space + without wrapping. If not, then we split. +- Check all of the elements of a tuple. Similarly to how arguments are + analyzed. This allows tuples to be split more rationally. +- Adjust splitting penalties around arithmetic operators so that the code can + flow more freely. The code must flow! +- Try to meld an argument list's closing parenthesis to the last argument. +### Fixed +- Attempt to determine if long lambdas are allowed. This can be done on a + case-by-case basis with a "pylint" disable comment. +- A comment before a decorator isn't part of the decorator's line. +- Only force a new wrapped line after a comment in a decorator when it's the + first token in the decorator. + +## [0.21.0] 2018-03-18 +### Added +- Introduce a new option of formatting multiline literals. Add + `SPLIT_BEFORE_CLOSING_BRACKET` knob to control whether closing bracket should + get their own line. +- Added `CONTINUATION_ALIGN_STYLE` knob to choose continuation alignment style + when `USE_TABS` is enabled. +- Add 'BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION' knob to control the number + of blank lines between top-level function and class definitions. +### Fixed +- Don't split ellipses. + +## [0.20.2] 2018-02-12 +### Changed +- Improve the speed at which files are excluded by ignoring them earlier. +- Allow dictionaries to stay on a single line if they only have one entry +### Fixed +- Use tabs when constructing a continuation line when `USE_TABS` is enabled. +- A dictionary entry may not end in a colon, but may be an "unpacking" + operation: `**foo`. Take that into accound and don't split after the + unpacking operator. + +## [0.20.1] 2018-01-13 +### Fixed +- Don't treat 'None' as a keyword if calling a function on it, like '__ne__()'. +- use_tabs=True always uses a single tab per indentation level; spaces are + used for aligning vertically after that. +- Relax the split of a paren at the end of an if statement. With + `dedent_closing_brackets` option requires that it be able to split there. + +## [0.20.0] 2017-11-14 +### Added +- Improve splitting of comprehensions and generators. Add + `SPLIT_PENALTY_COMPREHENSION` knob to control preference for keeping + comprehensions on a single line and `SPLIT_COMPLEX_COMPREHENSION` to enable + splitting each clause of complex comprehensions onto its own line. +### Changed +- Take into account a named function argument when determining if we should + split before the first argument in a function call. +- Split before the first argument in a function call if the arguments contain a + dictionary that doesn't fit on a single line. +- Improve splitting of elements in a tuple. We want to split if there's a + function call in the tuple that doesn't fit on the line. +### Fixed +- Enforce spaces between ellipses and keywords. +- When calculating the split penalty for a "trailer", process the child nodes + afterwards because their penalties may change. For example if a list + comprehension is an argument. +- Don't enforce a split before a comment after the opening of a container if it + doesn't it on the current line. We try hard not to move such comments around. +- Use a TextIOWrapper when reading from stdin in Python3. This is necessary for + some encodings, like cp936, used on Windows. +- Remove the penalty for a split before the first argument in a function call + where the only argument is a generator expression. + +## [0.19.0] 2017-10-14 +### Added +- Added `SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN` that enforces a split + after the opening paren of an expression that's surrounded by parens. +### Changed +- Split before the ending bracket of a comma-terminated tuple / argument list + if it's not a single element tuple / arg list. +### Fixed +- Prefer to split after a comma in an argument list rather than in the middle + of an argument. +- A non-multiline string may have newlines if it contains continuation markers + itself. Don't add a newline after the string when retaining the vertical + space. +- Take into account the "async" keyword when determining if we must split + before the first argument. +- Increase affinity for "atom" arguments in function calls. This helps prevent + lists from being separated when they don't need to be. +- Don't place a dictionary argument on its own line if it's the last argument + in the function call where that function is part of a builder-style call. +- Append the "var arg" type to a star in a star_expr. + +## [0.18.0] 2017-09-18 +### Added +- Option `ALLOW_SPLIT_BEFORE_DICT_VALUE` allows a split before a value. If + False, then it won't be split even if it goes over the column limit. +### Changed +- Use spaces around the '=' in a typed name argument to align with 3.6 syntax. +### Fixed +- Allow semicolons if the line is disabled. +- Fix issue where subsequent comments at decreasing levels of indentation + were improperly aligned and/or caused output with invalid syntax. +- Fix issue where specifying a line range removed a needed line before a + comment. +- Fix spacing between unary operators if one is 'not'. +- Indent the dictionary value correctly if there's a multi-line key. +- Don't remove needed spacing before a comment in a dict when in "chromium" + style. +- Increase indent for continuation line with same indent as next logical line + with 'async with' statement. + +## [0.17.0] 2017-08-20 +### Added +- Option `NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS` prevents adding spaces + around selected binary operators, in accordance with the current style guide. +### Changed +- Adjust blank lines on formatting boundaries when using the `--lines` option. +- Return 1 if a diff changed the code. This is in line with how GNU diff acts. +- Add `-vv` flag to print out file names as they are processed +### Fixed +- Corrected how `DEDENT_CLOSING_BRACKETS` and `COALESCE_BRACKETS` interacted. +- Fix return value to return a boolean. +- Correct vim plugin not to clobber edited code if yapf returns an error. +- Ensured comma-terminated tuples with multiple elements are split onto separate lines. + +## [0.16.3] 2017-07-13 +### Changed +- Add filename information to a ParseError exception. +### Fixed +- A token that ends in a continuation marker may have more than one newline in + it, thus changing its "lineno" value. This can happen if multiple + continuation markers are used with no intervening tokens. Adjust the line + number to account for the lines covered by those markers. +- Make sure to split after a comment even for "pseudo" parentheses. + +## [0.16.2] 2017-05-19 +### Fixed +- Treat expansion operators ('*', '**') in a similar way to function calls to + avoid splitting directly after the opening parenthesis. +- Increase the penalty for splitting after the start of a tuple. +- Increase penalty for excess characters. +- Check that we have enough children before trying to access them all. +- Remove trailing whitespaces from comments. +- Split before a function call in a list if the full list isn't able to fit on + a single line. +- Trying not to split around the '=' of a named assign. +- Changed split before the first argument behavior to ignore compound + statements like if and while, but not function declarations. +- Changed coalesce brackets not to line split before closing bracket. + +## [0.16.1] 2017-03-22 +### Changed +- Improved performance of cloning the format decision state object. This + improved the time in one *large* case from 273.485s to 234.652s. +- Relax the requirement that a named argument needs to be on one line. Going + over the column limit is more of an issue to pylint than putting named args + on multiple lines. +- Don't make splitting penalty decisions based on the original formatting. This + can and does lead to non-stable formatting, where yapf will reformat the same + code in different ways. +### Fixed +- Ensure splitting of arguments if there's a named assign present. +- Prefer to coalesce opening brackets if it's not at the beginning of a + function call. +- Prefer not to squish all of the elements in a function call over to the + right-hand side. Split the arguments instead. +- We need to split a dictionary value if the first element is a comment anyway, + so don't force the split here. It's forced elsewhere. +- Ensure tabs are used for continued indentation when USE_TABS is True. + +## [0.16.0] 2017-02-05 +### Added +- The `EACH_DICT_ENTRY_ON_SEPARATE_LINE` knob indicates that each dictionary + entry should be in separate lines if the full dictionary isn't able to fit on + a single line. +- The `SPLIT_BEFORE_DICT_SET_GENERATOR` knob splits before the `for` part of a + dictionary/set generator. +- The `BLANK_LINE_BEFORE_CLASS_DOCSTRING` knob adds a blank line before a + class's docstring. +- The `ALLOW_MULTILINE_DICTIONARY_KEYS` knob allows dictionary keys to span + more than one line. +### Fixed +- Split before all entries in a dict/set or list maker when comma-terminated, + even if there's only one entry. +- Will now try to set O_BINARY mode on stdout under Windows and Python 2. +- Avoid unneeded newline transformation when writing formatted code to + output on (affects only Python 2) + +## [0.15.2] 2017-01-29 +### Fixed +- Don't perform a global split when a named assign is part of a function call + which itself is an argument to a function call. I.e., don't cause 'a' to + split here: + + func(a, b, c, d(x, y, z=42)) +- Allow splitting inside a subscript if it's a logical or bitwise operating. + This should keep the subscript mostly contiguous otherwise. + +## [0.15.1] 2017-01-21 +### Fixed +- Don't insert a space between a type hint and the '=' sign. +- The '@' operator can be used in Python 3 for matrix multiplication. Give the + '@' in the decorator a DECORATOR subtype to distinguish it. +- Encourage the formatter to split at the beginning of an argument list instead + of in the middle. Especially if the middle is an empty parameter list. This + adjusts the affinity of binary and comparison operators. In particular, the + "not in" and other such operators don't want to have a split after it (or + before it) if at all possible. + +## [0.15.0] 2017-01-12 +### Added +- Keep type annotations intact as much as possible. Don't try to split the over + multiple lines. +### Fixed +- When determining if each element in a dictionary can fit on a single line, we + are skipping dictionary entries. However, we need to ignore comments in our + calculations and implicitly concatenated strings, which are already placed on + separate lines. +- Allow text before a "pylint" comment. +- Also allow text before a "yapf: (disable|enable)" comment. + +## [0.14.0] 2016-11-21 +### Added +- formatting can be run in parallel using the "-p" / "--parallel" flags. +### Fixed +- "not in" and "is not" should be subtyped as binary operators. +- A non-Node dictionary value may have a comment before it. In those cases, we + want to avoid encompassing only the comment in pseudo parens. So we include + the actual value as well. +- Adjust calculation so that pseudo-parentheses don't count towards the total + line length. +- Don't count a dictionary entry as not fitting on a single line in a + dictionary. +- Don't count pseudo-parentheses in the length of the line. + +## [0.13.2] 2016-10-22 +### Fixed +- REGRESSION: A comment may have a prefix with newlines in it. When calculating + the prefix indent, we cannot take the newlines into account. Otherwise, the + comment will be misplaced causing the code to fail. + +## [0.13.1] 2016-10-17 +### Fixed +- Correct emitting a diff that was accidentally removed. + +## [0.13.0] 2016-10-16 +### Added +- Added support to retain the original line endings of the source code. + +### Fixed +- Functions or classes with comments before them were reformatting the comments + even if the code was supposed to be ignored by the formatter. We now don't + adjust the whitespace before a function's comment if the comment is a + "disabled" line. We also don't count "# yapf: {disable|enable}" as a disabled + line, which seems logical. +- It's not really more readable to split before a dictionary value if it's part + of a dictionary comprehension. +- Enforce two blank lines after a function or class definition, even before a + comment. (But not between a decorator and a comment.) This is related to PEP8 + error E305. +- Remove O(n^2) algorithm from the line disabling logic. + +## [0.12.2] 2016-10-09 +### Fixed +- If `style.SetGlobalStyle(<create pre-defined style>)` was called and then + `yapf_api.FormatCode` was called, the style set by the first call would be + lost, because it would return the style created by `DEFAULT_STYLE_FACTORY`, + which is set to PEP8 by default. Fix this by making the first call set which + factory we call as the "default" style. +- Don't force a split before non-function call arguments. +- A dictionary being used as an argument to a function call and which can exist + on a single line shouldn't be split. +- Don't rely upon the original line break to determine if we should split + before the elements in a container. Especially split if there's a comment in + the container. +- Don't add spaces between star and args in a lambda expression. +- If a nested data structure terminates in a comma, then split before the first + element, but only if there's more than one element in the list. + +## [0.12.1] 2016-10-02 +### Changed +- Dictionary values will be placed on the same line as the key if *all* of the + elements in the dictionary can be placed on one line. Otherwise, the + dictionary values will be placed on the next line. + +### Fixed +- Prefer to split before a terminating r-paren in an argument list if the line + would otherwise go over the column limit. +- Split before the first key in a dictionary if the dictionary cannot fit on a + single line. +- Don't count "pylint" comments when determining if the line goes over the + column limit. +- Don't count the argument list of a lambda as a named assign in a function + call. + +## [0.12.0] 2016-09-25 +### Added +- Support formatting of typed names. Typed names are formatted a similar way to + how named arguments are formatted, except that there's a space after the + colon. +- Add a knob, 'SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN', to allow adding spaces + around the assign operator on default or named assigns. + +## Changed +- Turn "verification" off by default for external APIs. +- If a function call in an argument list won't fit on the current line but will + fit on a line by itself, then split before the call so that it won't be split + up unnecessarily. + +## Fixed +- Don't add space after power operator if the next operator's a unary operator. + +## [0.11.1] 2016-08-17 +### Changed +- Issue #228: Return exit code 0 on success, regardless of whether files were + changed. (Previously, 0 meant success with no files + modified, and 2 meant success with at least one file modified.) + +### Fixed +- Enforce splitting each element in a dictionary if comma terminated. +- It's okay to split in the middle of a dotted name if the whole expression is + going to go over the column limit. +- Asynchronous functions were going missing if they were preceded by a comment + (a what? exactly). The asynchronous function processing wasn't taking the + comment into account and thus skipping the whole function. +- The splitting of arguments when comma terminated had a conflict. The split + penalty of the closing bracket was set to the maximum, but it shouldn't be if + the closing bracket is preceded by a comma. + +## [0.11.0] 2016-07-17 +### Added +- The COALESCE_BRACKETS knob prevents splitting consecutive brackets when + DEDENT_CLOSING_BRACKETS is set. +- Don't count "pylint" directives as exceeding the column limit. + +### Changed +- We split all of the arguments to a function call if there's a named argument. + In this case, we want to split after the opening bracket too. This makes + things look a bit better. + +### Fixed +- When retaining format of a multiline string with Chromium style, make sure + that the multiline string doesn't mess up where the following comma ends up. +- Correct for when 'lib2to3' smooshes comments together into the same DEDENT + node. + +## [0.10.0] 2016-06-14 +### Added +- Add a knob, 'USE_TABS', to allow using tabs for indentation. + +### Changed +- Performance enhancements. + +### Fixed +- Don't split an import list if it's not surrounded by parentheses. + +## [0.9.0] 2016-05-29 +### Added +- Added a knob (SPLIT_PENALTY_BEFORE_IF_EXPR) to adjust the split penalty + before an if expression. This allows the user to place a list comprehension + all on one line. +- Added a knob (SPLIT_BEFORE_FIRST_ARGUMENT) that encourages splitting before + the first element of a list of arguments or parameters if they are going to + be split anyway. +- Added a knob (SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED) splits arguments to a + function if the list is terminated by a comma. + +### Fixed +- Don't split before a first element list argument as we would before a first + element function call. +- Don't penalize when we must split a line. +- Allow splitting before the single argument in a function call. + +## [0.8.2] 2016-05-21 +### Fixed +- Prefer not to split after the opening of a subscript. +- Don't add space before the 'await' keyword if it's preceded by an opening + paren. +- When we're setting the split penalty for a continuous list, we don't want to + mistake a comment at the end of that list as part of the list. +- When calculating blank lines, don't assume the last seen object was a class + or function when we're in a class or function. +- Don't count the closing scope when determining if the current scope is the + last scope on the line. + +## [0.8.1] 2016-05-18 +### Fixed +- 'SPLIT_BEFORE_LOGICAL_OPERATOR' wasn't working correctly. The penalty was + being set incorrectly when it was part of a larger construct. +- Don't separate a keyword, like "await", from a left paren. +- Don't rely upon the original tokens' line number to determine if we should + perform splitting in Facebook mode. The line number isn't the line number of + the reformatted token, but the line number where it was in the original code. + Instead, we need to carefully determine if the line is liabel to be split and + act accordingly. + +## [0.8.0] 2016-05-10 +### Added +- Add a knob, 'SPACES_AROUND_POWER_OPERATOR', to allow adding spaces around the + power operator. + +### Fixed +- There shouldn't be a space between a decorator and an intervening comment. +- If we split before a bitwise operator, then we assume that the programmer + knows what they're doing, more or less, and so we enforce a split before said + operator if one exists in the original program. +- Strengthen the bond between a keyword and value argument. +- Don't add a blank line after a multiline string. +- If the "for" part of a list comprehension can exist on the starting line + without going over the column limit, then let it remain there. + +## [0.7.1] 2016-04-21 +### Fixed +- Don't rewrite the file if there are no changes. +- Ensure the proper number of blank lines before an async function. +- Split after a bitwise operator when in PEP 8 mode. +- Retain the splitting within a dictionary data literal between the key and + value. +- Try to keep short function calls all on one line even if they're part of a + larger series of tokens. This stops us from splitting too much. + +## [0.7.0] 2016-04-09 +### Added +- Support for Python 3.5. +- Add 'ALLOW_MULTILINE_LAMBDAS' which allows lambdas to be formatted onto + multiple lines. + +### Fixed +- Lessen penalty for splitting before a dictionary keyword. +- Formatting of trailing comments on disabled formatting lines. +- Disable / enable formatting at end of multi-line comment. + +## [0.6.3] 2016-03-06 +### Changed +- Documentation updated. + +### Fixed +- Fix spacing of multiline comments when formatting is disabled. + +## [0.6.2] 2015-11-01 +### Changed +- Look at the 'setup.cfg' file to see if it contains style information for + YAPF. +- Look at the '~/.config/yapf/style' file to see if it contains global style + information for YAPF. + +### Fixed +- Make lists that can fit on one line more likely to stay together. +- Correct formatting of '*args' and '**kwargs' when there are default values in + the argument list. + +## [0.6.1] 2015-10-24 +### Fixed +- Make sure to align comments in data literals correctly. Also make sure we + don't count a "#." in a string as an i18n comment. +- Retain proper vertical spacing before comments in a data literal. +- Make sure that continuations from a compound statement are distinguished from + the succeeding line. +- Ignore preceding comments when calculating what is a "dictonary maker". +- Add a small penalty for splitting before a closing bracket. +- Ensure that a space is enforced after we remove a pseudo-paren that's between + two names, keywords, numbers, etc. +- Increase the penalty for splitting after a pseudo-paren. This could lead to + less readable code in some circumstances. + +## [0.6.0] 2015-10-18 +### Added +- Add knob to indent the dictionary value if there is a split before it. + +### Changed +- No longer check that a file is a "Python" file unless the '--recursive' flag + is specified. +- No longer allow the user to specify a directory unless the '--recursive' flag + is specified. + +### Fixed +- When determining if we should split a dictionary's value to a new line, use + the longest entry instead of the total dictionary's length. This allows the + formatter to reformat the dictionary in a more consistent manner. +- Improve how list comprehensions are formatted. Make splitting dependent upon + whether the "comp_for" or "comp_if" goes over the column limit. +- Don't over indent if expression hanging indents if we expect to dedent the + closing bracket. +- Improve splitting heuristic when the first argument to a function call is + itself a function call with arguments. In cases like this, the remaining + arguments to the function call would look badly aligned, even though they are + techincally correct (the best kind of correct!). +- Improve splitting heuristic more so that if the first argument to a function + call is a data literal that will go over the column limit, then we want to + split before it. +- Remove spaces around '**' operator. +- Retain formatting of comments in the middle of an expression. +- Don't add a newline to an empty file. +- Over indent a function's parameter list if it's not distinguished from the + body of the function. + +## [0.5.0] 2015-10-11 +### Added +- Add option to exclude files/directories from formatting. +- Add a knob to control whether import names are split after the first '('. + +### Fixed +- Indent the continuation of an if-then statement when it's not distinguished + from the body of the if-then. +- Allow for sensible splitting of array indices where appropriate. +- Prefer to not split before the ending bracket of an atom. This produces + better code in most cases. +- Corrected how horizontal spaces were presevered in a disabled region. + +## [0.4.0] 2015-10-07 +### Added +- Support for dedenting closing brackets, "facebook" style. + +### Fixed +- Formatting of tokens after a multiline string didn't retain their horizontal + spacing. + +## [0.3.1] 2015-09-30 +### Fixed +- Format closing scope bracket correctly when indentation size changes. + +## [0.3.0] 2015-09-20 +### Added +- Return a 2 if the source changed, 1 on error, and 0 for no change. + +### Fixed +- Make sure we format if the "lines" specified are in the middle of a + statement. + +## [0.2.9] - 2015-09-13 +### Fixed +- Formatting of multiple files. It was halting after formatting the first file. + +## [0.2.8] - 2015-09-12 +### Added +- Return a non-zero exit code if the source was changed. +- Add bitwise operator splitting penalty and prefer to split before bitwise + operators. + +### Fixed +- Retain vertical spacing between disabled and enabled lines. +- Split only at start of named assign. +- Retain comment position when formatting is disabled. +- Honor splitting before or after logical ops. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..0b113c0 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,44 @@ +Want to contribute? Great! First, read this page (including the small print at the end). + +Before you contribute +--------------------- + +Before we can use your code, you must sign the `Google Individual Contributor +License Agreement +<https://developers.google.com/open-source/cla/individual?csw=1>`_ (CLA), which +you can do online. The CLA is necessary mainly because you own the +copyright to your changes, even after your contribution becomes part of our +codebase, so we need your permission to use and distribute your code. We also +need to be sure of various other things—for instance that you'll tell us if you +know that your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. +Before you start working on a larger contribution, you should get in touch with +us first through the issue tracker with your idea so that we can help out and +possibly guide you. Coordinating up front makes it much easier to avoid +frustration later on. + +Code reviews +------------ + +All submissions, including submissions by project members, require review. We +use Github pull requests for this purpose. + +YAPF coding style +----------------- + +YAPF follows the `Chromium Python Style Guide +<https://www.chromium.org/chromium-os/python-style-guidelines>`_. It's the same +as the Google Python Style guide with two exceptions: + +- 2 spaces for indentation rather than 4. +- CamelCase for function and method names rather than words_with_underscores. + +The rationale for this is that YAPF was initially developed at Google where +these two exceptions are still part of the internal Python style guide. + +Small print +----------- + +Contributions made by corporations are covered by a different agreement than +the one above, the Software Grant and Corporate Contributor License Agreement. diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..62e2a5f --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,15 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# Names should be added to this file as: +# Name <email address> + +Bill Wendling <morbo@google.com> +Eli Bendersky <eliben@google.com> +Sam Clegg <sbc@google.com> +Łukasz Langa <ambv@fb.com> diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..cc27c5a --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,32 @@ +Running YAPF on itself +---------------------- + +To run YAPF on all of YAPF:: + + $ PYTHONPATH=$PWD/yapf python -m yapf -i -r . + +To run YAPF on just the files changed in the current git branch:: + + $ PYTHONPATH=$PWD/yapf python -m yapf -i $(git diff --name-only @{upstream}) + +Releasing a new version +----------------------- + +* Run tests: python setup.py test + [don't forget to run with Python 2.7 and 3.6] + +* Bump version in yapf/__init__.py + +* Build source distribution: python setup.py sdist + +* Check it looks OK, install it onto a virtualenv, run tests, run yapf as a tool + +* Build release: python setup.py sdist bdist_wheel + +* Push to PyPI: twine upload dist/* + +* Test in a clean virtualenv that 'pip install yapf' works with the new version + +* Commit the version bump; add tag with git tag v<VERSION_NUM>; git push --tags + +TODO: discuss how to use tox to make virtualenv testing easier. @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5c70a55 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include HACKING.rst LICENSE AUTHORS CHANGELOG CONTRIBUTING.rst CONTRIBUTORS +include .coveragerc .editorconfig .flake8 plugins/README.rst +include plugins/vim/autoload/yapf.vim plugins/vim/plugin/yapf.vim pylintrc +include .style.yapf tox.ini .travis.yml .vimrc diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..19ef0c2 --- /dev/null +++ b/README.rst @@ -0,0 +1,710 @@ +==== +YAPF +==== + +.. image:: https://badge.fury.io/py/yapf.svg + :target: https://badge.fury.io/py/yapf + :alt: PyPI version + +.. image:: https://travis-ci.org/google/yapf.svg?branch=master + :target: https://travis-ci.org/google/yapf + :alt: Build status + +.. image:: https://coveralls.io/repos/google/yapf/badge.svg?branch=master + :target: https://coveralls.io/r/google/yapf?branch=master + :alt: Coverage status + + +Introduction +============ + +Most of the current formatters for Python --- e.g., autopep8, and pep8ify --- +are made to remove lint errors from code. This has some obvious limitations. +For instance, code that conforms to the PEP 8 guidelines may not be +reformatted. But it doesn't mean that the code looks good. + +YAPF takes a different approach. It's based off of 'clang-format', developed by +Daniel Jasper. In essence, the algorithm takes the code and reformats it to the +best formatting that conforms to the style guide, even if the original code +didn't violate the style guide. The idea is also similar to the 'gofmt' tool for +the Go programming language: end all holy wars about formatting - if the whole +codebase of a project is simply piped through YAPF whenever modifications are +made, the style remains consistent throughout the project and there's no point +arguing about style in every code review. + +The ultimate goal is that the code YAPF produces is as good as the code that a +programmer would write if they were following the style guide. It takes away +some of the drudgery of maintaining your code. + +Try out YAPF with this `online demo <https://yapf.now.sh>`_. + +.. footer:: + + YAPF is not an official Google product (experimental or otherwise), it is + just code that happens to be owned by Google. + +.. contents:: + + +Installation +============ + +To install YAPF from PyPI: + +.. code-block:: shell + + $ pip install yapf + +(optional) If you are using Python 2.7 and want to enable multiprocessing: + +.. code-block:: shell + + $ pip install futures + +YAPF is still considered in "alpha" stage, and the released version may change +often; therefore, the best way to keep up-to-date with the latest development +is to clone this repository. + +Note that if you intend to use YAPF as a command-line tool rather than as a +library, installation is not necessary. YAPF supports being run as a directory +by the Python interpreter. If you cloned/unzipped YAPF into ``DIR``, it's +possible to run: + +.. code-block:: shell + + $ PYTHONPATH=DIR python DIR/yapf [options] ... + + +Python versions +=============== + +YAPF supports Python 2.7 and 3.6.4+. (Note that some Python 3 features may fail +to parse with Python versions before 3.6.4.) + +YAPF requires the code it formats to be valid Python for the version YAPF itself +runs under. Therefore, if you format Python 3 code with YAPF, run YAPF itself +under Python 3 (and similarly for Python 2). + + +Usage +===== + +Options:: + + usage: yapf [-h] [-v] [-d | -i] [-r | -l START-END] [-e PATTERN] + [--style STYLE] [--style-help] [--no-local-style] [-p] + [-vv] + [files [files ...]] + + Formatter for Python code. + + positional arguments: + files + + optional arguments: + -h, --help show this help message and exit + -v, --version show version number and exit + -d, --diff print the diff for the fixed source + -i, --in-place make changes to files in place + -r, --recursive run recursively over directories + -l START-END, --lines START-END + range of lines to reformat, one-based + -e PATTERN, --exclude PATTERN + patterns for files to exclude from formatting + --style STYLE specify formatting style: either a style name (for + example "pep8" or "google"), or the name of a file + with style settings. The default is pep8 unless a + .style.yapf or setup.cfg file located in the same + directory as the source or one of its parent + directories (for stdin, the current directory is + used). + --style-help show style settings and exit; this output can be saved + to .style.yapf to make your settings permanent + --no-local-style don't search for local style definition + -p, --parallel Run yapf in parallel when formatting multiple files. + Requires concurrent.futures in Python 2.X + -vv, --verbose Print out file names while processing + + +------------ +Return Codes +------------ + +Normally YAPF returns zero on successful program termination and non-zero otherwise. + +If ``--diff`` is supplied, YAPF returns zero when no changes were necessary, non-zero +otherwise (including program error). You can use this in a CI workflow to test that code +has been YAPF-formatted. + + +Formatting style +================ + +The formatting style used by YAPF is configurable and there are many "knobs" +that can be used to tune how YAPF does formatting. See the ``style.py`` module +for the full list. + +To control the style, run YAPF with the ``--style`` argument. It accepts one of +the predefined styles (e.g., ``pep8`` or ``google``), a path to a configuration +file that specifies the desired style, or a dictionary of key/value pairs. + +The config file is a simple listing of (case-insensitive) ``key = value`` pairs +with a ``[style]`` heading. For example: + +.. code-block:: ini + + [style] + based_on_style = pep8 + spaces_before_comment = 4 + split_before_logical_operator = true + +The ``based_on_style`` setting determines which of the predefined styles this +custom style is based on (think of it like subclassing). + +It's also possible to do the same on the command line with a dictionary. For +example: + +.. code-block:: shell + + --style='{based_on_style: chromium, indent_width: 4}' + +This will take the ``chromium`` base style and modify it to have four space +indentations. + +YAPF will search for the formatting style in the following manner: + +1. Specified on the command line +2. In the `[style]` section of a `.style.yapf` file in either the current + directory or one of its parent directories. +3. In the `[yapf]` section of a `setup.cfg` file in either the current + directory or one of its parent directories. +4. In the `~/.config/yapf/style` file in your home directory. + +If none of those files are found, the default style is used (PEP8). + + +Example +======= + +An example of the type of formatting that YAPF can do, it will take this ugly +code: + +.. code-block:: python + + x = { 'a':37,'b':42, + + 'c':927} + + y = 'hello ''world' + z = 'hello '+'world' + a = 'hello {}'.format('world') + class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y + def f ( a ) : + return 37+-+a[42-x : y**3] + +and reformat it into: + +.. code-block:: python + + x = {'a': 37, 'b': 42, 'c': 927} + + y = 'hello ' 'world' + z = 'hello ' + 'world' + a = 'hello {}'.format('world') + + + class foo(object): + def f(self): + return 37 * -+2 + + def g(self, x, y=42): + return y + + + def f(a): + return 37 + -+a[42 - x:y**3] + + +Example as a module +=================== + +The two main APIs for calling yapf are ``FormatCode`` and ``FormatFile``, these +share several arguments which are described below: + +.. code-block:: python + + >>> from yapf.yapflib.yapf_api import FormatCode # reformat a string of code + + >>> FormatCode("f ( a = 1, b = 2 )") + 'f(a=1, b=2)\n' + +A ``style_config`` argument: Either a style name or a path to a file that contains +formatting style settings. If None is specified, use the default style +as set in ``style.DEFAULT_STYLE_FACTORY``. + +.. code-block:: python + + >>> FormatCode("def g():\n return True", style_config='pep8') + 'def g():\n return True\n' + +A ``lines`` argument: A list of tuples of lines (ints), [start, end], +that we want to format. The lines are 1-based indexed. It can be used by +third-party code (e.g., IDEs) when reformatting a snippet of code rather +than a whole file. + +.. code-block:: python + + >>> FormatCode("def g( ):\n a=1\n b = 2\n return a==b", lines=[(1, 1), (2, 3)]) + 'def g():\n a = 1\n b = 2\n return a==b\n' + +A ``print_diff`` (bool): Instead of returning the reformatted source, return a +diff that turns the formatted source into reformatter source. + +.. code-block:: python + + >>> print(FormatCode("a==b", filename="foo.py", print_diff=True)) + --- foo.py (original) + +++ foo.py (reformatted) + @@ -1 +1 @@ + -a==b + +a == b + +Note: the ``filename`` argument for ``FormatCode`` is what is inserted into +the diff, the default is ``<unknown>``. + +``FormatFile`` returns reformatted code from the passed file along with its encoding: + +.. code-block:: python + + >>> from yapf.yapflib.yapf_api import FormatFile # reformat a file + + >>> print(open("foo.py").read()) # contents of file + a==b + + >>> FormatFile("foo.py") + ('a == b\n', 'utf-8') + +The ``in-place`` argument saves the reformatted code back to the file: + +.. code-block:: python + + >>> FormatFile("foo.py", in_place=True) + (None, 'utf-8') + + >>> print(open("foo.py").read()) # contents of file (now fixed) + a == b + + +Knobs +===== + +``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` + Align closing bracket with visual indentation. + +``ALLOW_MULTILINE_LAMBDAS`` + Allow lambdas to be formatted on more than one line. + +``ALLOW_MULTILINE_DICTIONARY_KEYS`` + Allow dictionary keys to exist on multiple lines. For example: + + .. code-block:: python + + x = { + ('this is the first element of a tuple', + 'this is the second element of a tuple'): + value, + } + +``ALLOW_SPLIT_BEFORE_DICT_VALUE`` + Allow splits before the dictionary value. + +``BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF`` + Insert a blank line before a ``def`` or ``class`` immediately nested within + another ``def`` or ``class``. For example: + + .. code-block:: python + + class Foo: + # <------ this blank line + def method(): + pass + +``BLANK_LINE_BEFORE_MODULE_DOCSTRING`` + Insert a blank line before a module docstring. + +``BLANK_LINE_BEFORE_CLASS_DOCSTRING`` + Insert a blank line before a class-level docstring. + +``BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION`` + Sets the number of desired blank lines surrounding top-level function and + class definitions. For example: + + .. code-block:: python + + class Foo: + pass + # <------ having two blank lines here + # <------ is the default setting + class Bar: + pass + +``COALESCE_BRACKETS`` + Do not split consecutive brackets. Only relevant when + ``DEDENT_CLOSING_BRACKETS`` is set. For example: + + .. code-block:: python + + call_func_that_takes_a_dict( + { + 'key1': 'value1', + 'key2': 'value2', + } + ) + + would reformat to: + + .. code-block:: python + + call_func_that_takes_a_dict({ + 'key1': 'value1', + 'key2': 'value2', + }) + + +``COLUMN_LIMIT`` + The column limit (or max line-length) + +``CONTINUATION_ALIGN_STYLE`` + The style for continuation alignment. Possible values are: + + - SPACE: Use spaces for continuation alignment. This is default behavior. + - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns + (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation + alignment. + - VALIGN-RIGHT: Vertically align continuation lines with indent characters. + Slightly right (one more indent character) if cannot vertically align + continuation lines with indent characters. + + For options ``FIXED``, and ``VALIGN-RIGHT`` are only available when + ``USE_TABS`` is enabled. + +``CONTINUATION_INDENT_WIDTH`` + Indent width used for line continuations. + +``DEDENT_CLOSING_BRACKETS`` + Put closing brackets on a separate line, dedented, if the bracketed + expression can't fit in a single line. Applies to all kinds of brackets, + including function definitions and calls. For example: + + .. code-block:: python + + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is dedented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is dedented and on a separate line + +``DISABLE_ENDING_COMMA_HEURISTIC`` + Disable the heuristic which places each list element on a separate line if + the list is comma-terminated. + +``EACH_DICT_ENTRY_ON_SEPARATE_LINE`` + Place each dictionary entry onto its own line. + +``I18N_COMMENT`` + The regex for an internationalization comment. The presence of this comment + stops reformatting of that line, because the comments are required to be + next to the string they translate. + +``I18N_FUNCTION_CALL`` + The internationalization function call names. The presence of this function + stops reformatting on that line, because the string it has cannot be moved + away from the i18n comment. + +``INDENT_DICTIONARY_VALUE`` + Indent the dictionary value if it cannot fit on the same line as the + dictionary key. For example: + + .. code-block:: python + + config = { + 'key1': + 'value1', + 'key2': value1 + + value2, + } + +``INDENT_WIDTH`` + The number of columns to use for indentation. + +``JOIN_MULTIPLE_LINES`` + Join short lines into one line. E.g., single line ``if`` statements. + +``SPACES_AROUND_POWER_OPERATOR`` + Set to ``True`` to prefer using spaces around ``**``. + +``NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS`` + Do not include spaces around selected binary operators. For example: + + .. code-block:: python + + 1 + 2 * 3 - 4 / 5 + + will be formatted as follows when configured with ``*,/``: + + .. code-block:: python + + 1 + 2*3 - 4/5 + +``SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN`` + Set to ``True`` to prefer spaces around the assignment operator for default + or keyword arguments. + +``SPACES_BEFORE_COMMENT`` + The number of spaces required before a trailing comment. + +``SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET`` + Insert a space between the ending comma and closing bracket of a list, etc. + +``SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED`` + Split before arguments if the argument list is terminated by a comma. + +``SPLIT_ALL_COMMA_SEPARATED_VALUES`` + If a comma separated list (dict, list, tuple, or function def) is on a + line that is too long, split such that all elements are on a single line. + +``SPLIT_BEFORE_BITWISE_OPERATOR`` + Set to ``True`` to prefer splitting before ``&``, ``|`` or ``^`` rather + than after. + +``SPLIT_BEFORE_CLOSING_BRACKET`` + Split before the closing bracket if a list or dict literal doesn't fit on + a single line. + +``SPLIT_BEFORE_DICT_SET_GENERATOR`` + Split before a dictionary or set generator (comp_for). For example, note + the split before the ``for``: + + .. code-block:: python + + foo = { + variable: 'Hello world, have a nice day!' + for variable in bar if variable != 42 + } + +``SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN`` + Split after the opening paren which surrounds an expression if it doesn't + fit on a single line. + +``SPLIT_BEFORE_FIRST_ARGUMENT`` + If an argument / parameter list is going to be split, then split before the + first argument. + +``SPLIT_BEFORE_LOGICAL_OPERATOR`` + Set to ``True`` to prefer splitting before ``and`` or ``or`` rather than + after. + +``SPLIT_BEFORE_NAMED_ASSIGNS`` + Split named assignments onto individual lines. + +``SPLIT_COMPLEX_COMPREHENSION`` + For list comprehensions and generator expressions with multiple clauses + (e.g multiple "for" calls, "if" filter expressions) and which need to be + reflowed, split each clause onto its own line. For example: + + .. code-block:: python + + result = [ + a_var + b_var for a_var in xrange(1000) for b_var in xrange(1000) + if a_var % b_var] + + would reformat to something like: + + .. code-block:: python + + result = [ + a_var + b_var + for a_var in xrange(1000) + for b_var in xrange(1000) + if a_var % b_var] + +``SPLIT_PENALTY_AFTER_OPENING_BRACKET`` + The penalty for splitting right after the opening bracket. + +``SPLIT_PENALTY_AFTER_UNARY_OPERATOR`` + The penalty for splitting the line after a unary operator. + +``SPLIT_PENALTY_BEFORE_IF_EXPR`` + The penalty for splitting right before an ``if`` expression. + +``SPLIT_PENALTY_BITWISE_OPERATOR`` + The penalty of splitting the line around the ``&``, ``|``, and ``^`` + operators. + +``SPLIT_PENALTY_COMPREHENSION`` + The penalty for splitting a list comprehension or generator expression. + +``SPLIT_PENALTY_EXCESS_CHARACTER`` + The penalty for characters over the column limit. + +``SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT`` + The penalty incurred by adding a line split to the unwrapped line. The more + line splits added the higher the penalty. + +``SPLIT_PENALTY_IMPORT_NAMES`` + The penalty of splitting a list of ``import as`` names. For example: + + .. code-block:: python + + from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, + long_argument_2, + long_argument_3) + + would reformat to something like: + + .. code-block:: python + + from a_very_long_or_indented_module_name_yada_yad import ( + long_argument_1, long_argument_2, long_argument_3) + +``SPLIT_PENALTY_LOGICAL_OPERATOR`` + The penalty of splitting the line around the ``and`` and ``or`` operators. + +``USE_TABS`` + Use the Tab character for indentation. + +(Potentially) Frequently Asked Questions +======================================== + +-------------------------------------------- +Why does YAPF destroy my awesome formatting? +-------------------------------------------- + +YAPF tries very hard to get the formatting correct. But for some code, it won't +be as good as hand-formatting. In particular, large data literals may become +horribly disfigured under YAPF. + +The reasons for this are manyfold. In short, YAPF is simply a tool to help +with development. It will format things to coincide with the style guide, but +that may not equate with readability. + +What can be done to alleviate this situation is to indicate regions YAPF should +ignore when reformatting something: + +.. code-block:: python + + # yapf: disable + FOO = { + # ... some very large, complex data literal. + } + + BAR = [ + # ... another large data literal. + ] + # yapf: enable + +You can also disable formatting for a single literal like this: + +.. code-block:: python + + BAZ = { + (1, 2, 3, 4), + (5, 6, 7, 8), + (9, 10, 11, 12), + } # yapf: disable + +To preserve the nice dedented closing brackets, use the +``dedent_closing_brackets`` in your style. Note that in this case all +brackets, including function definitions and calls, are going to use +that style. This provides consistency across the formatted codebase. + +------------------------------- +Why Not Improve Existing Tools? +------------------------------- + +We wanted to use clang-format's reformatting algorithm. It's very powerful and +designed to come up with the best formatting possible. Existing tools were +created with different goals in mind, and would require extensive modifications +to convert to using clang-format's algorithm. + +----------------------------- +Can I Use YAPF In My Program? +----------------------------- + +Please do! YAPF was designed to be used as a library as well as a command line +tool. This means that a tool or IDE plugin is free to use YAPF. + + +Gory Details +============ + +---------------- +Algorithm Design +---------------- + +The main data structure in YAPF is the ``UnwrappedLine`` object. It holds a list +of ``FormatToken``\s, that we would want to place on a single line if there were +no column limit. An exception being a comment in the middle of an expression +statement will force the line to be formatted on more than one line. The +formatter works on one ``UnwrappedLine`` object at a time. + +An ``UnwrappedLine`` typically won't affect the formatting of lines before or +after it. There is a part of the algorithm that may join two or more +``UnwrappedLine``\s into one line. For instance, an if-then statement with a +short body can be placed on a single line: + +.. code-block:: python + + if a == 42: continue + +YAPF's formatting algorithm creates a weighted tree that acts as the solution +space for the algorithm. Each node in the tree represents the result of a +formatting decision --- i.e., whether to split or not to split before a token. +Each formatting decision has a cost associated with it. Therefore, the cost is +realized on the edge between two nodes. (In reality, the weighted tree doesn't +have separate edge objects, so the cost resides on the nodes themselves.) + +For example, take the following Python code snippet. For the sake of this +example, assume that line (1) violates the column limit restriction and needs to +be reformatted. + +.. code-block:: python + + def xxxxxxxxxxx(aaaaaaaaaaaa, bbbbbbbbb, cccccccc, dddddddd, eeeeee): # 1 + pass # 2 + +For line (1), the algorithm will build a tree where each node (a +``FormattingDecisionState`` object) is the state of the line at that token given +the decision to split before the token or not. Note: the ``FormatDecisionState`` +objects are copied by value so each node in the graph is unique and a change in +one doesn't affect other nodes. + +Heuristics are used to determine the costs of splitting or not splitting. +Because a node holds the state of the tree up to a token's insertion, it can +easily determine if a splitting decision will violate one of the style +requirements. For instance, the heuristic is able to apply an extra penalty to +the edge when not splitting between the previous token and the one being added. + +There are some instances where we will never want to split the line, because +doing so will always be detrimental (i.e., it will require a backslash-newline, +which is very rarely desirable). For line (1), we will never want to split the +first three tokens: ``def``, ``xxxxxxxxxxx``, and ``(``. Nor will we want to +split between the ``)`` and the ``:`` at the end. These regions are said to be +"unbreakable." This is reflected in the tree by there not being a "split" +decision (left hand branch) within the unbreakable region. + +Now that we have the tree, we determine what the "best" formatting is by finding +the path through the tree with the lowest cost. + +And that's it! diff --git a/plugins/README.rst b/plugins/README.rst new file mode 100644 index 0000000..f7657cc --- /dev/null +++ b/plugins/README.rst @@ -0,0 +1,115 @@ +=========== +IDE Plugins +=========== + +Emacs +===== + +The ``Emacs`` plugin is maintained separately. Installation directions can be +found here: https://github.com/paetzke/py-yapf.el + +VIM +=== + +The ``vim`` plugin allows you to reformat a range of code. Copy ``plugin`` and +``autoload`` directories into your ~/.vim or use ``:packadd`` in Vim 8. Or use +a plugin manager like Plug or Vundle: + +.. code-block:: vim + + " Plug + Plug 'google/yapf', { 'rtp': 'plugins/vim', 'for': 'python' } + + " Vundle + Plugin 'google/yapf', { 'rtp': 'plugins/vim' } + + +You can add key bindings in the ``.vimrc`` file: + +.. code-block:: vim + + map <C-Y> :call yapf#YAPF()<cr> + imap <C-Y> <c-o>:call yapf#YAPF()<cr> + +Alternatively, you can call the command ``YAPF``. If you omit the range, it +will reformat the whole buffer. + +example: + +.. code-block:: vim + + :YAPF " formats whole buffer + :'<,'>YAPF " formats lines selected in visual mode + +Sublime Text +============ + +The ``Sublime Text`` plugin is also maintained separately. It is compatible +with both Sublime Text 2 and 3. + +The plugin can be easily installed by using *Sublime Package Control*. Check +the project page of the plugin for more information: +https://github.com/jason-kane/PyYapf + +=================== +git Pre-Commit Hook +=================== + +The ``git`` pre-commit hook automatically formats your Python files before they +are committed to your local repository. Any changes ``yapf`` makes to the files +will stay unstaged so that you can diff them manually. + +To install, simply download the raw file and copy it into your git hooks +directory: + +.. code-block:: bash + + # From the root of your git project. + curl -o pre-commit.sh https://raw.githubusercontent.com/google/yapf/master/plugins/pre-commit.sh + chmod a+x pre-commit.sh + mv pre-commit.sh .git/hooks/pre-commit + +========== +Textmate 2 +========== + +Plugin for ``Textmate 2`` requires ``yapf`` Python package installed on your +system: + +.. code-block:: shell + + pip install yapf + +Also, you will need to activate ``Python`` bundle from ``Preferences >> +Bundles``. + +Finally, create a ``~/Library/Application +Support/TextMate/Bundles/Python.tmbundle/Commands/YAPF.tmCommand`` file with +the following content: + +.. code-block:: xml + + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> + <plist version="1.0"> + <dict> + <key>beforeRunningCommand</key> + <string>saveActiveFile</string> + <key>command</key> + <string>#!/bin/bash + + TPY=${TM_PYTHON:-python} + + "$TPY" "/usr/local/bin/yapf" "$TM_FILEPATH"</string> + <key>input</key> + <string>document</string> + <key>name</key> + <string>YAPF</string> + <key>scope</key> + <string>source.python</string> + <key>uuid</key> + <string>297D5A82-2616-4950-9905-BD2D1C94D2D4</string> + </dict> + </plist> + +You will see a new menu item ``Bundles > Python > YAPF``. diff --git a/plugins/pre-commit.sh b/plugins/pre-commit.sh new file mode 100644 index 0000000..896f0ff --- /dev/null +++ b/plugins/pre-commit.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Git pre-commit hook to check staged Python files for formatting issues with +# yapf. +# +# INSTALLING: Copy this script into `.git/hooks/pre-commit`, and mark it as +# executable. +# +# This requires that yapf is installed and runnable in the environment running +# the pre-commit hook. +# +# When running, this first checks for unstaged changes to staged files, and if +# there are any, it will exit with an error. Files with unstaged changes will be +# printed. +# +# If all staged files have no unstaged changes, it will run yapf against them, +# leaving the formatting changes unstaged. Changed files will be printed. +# +# BUGS: This does not leave staged changes alone when used with the -a flag to +# git commit, due to the fact that git stages ALL unstaged files when that flag +# is used. + +# Find all staged Python files, and exit early if there aren't any. +PYTHON_FILES=(`git diff --name-only --cached --diff-filter=AM | \ + grep --color=never '.py$'`) +if [ ! "$PYTHON_FILES" ]; then + exit 0 +fi + +########## PIP VERSION ############# +# Verify that yapf is installed; if not, warn and exit. +if [ -z $(which yapf) ]; then + echo 'yapf not on path; can not format. Please install yapf:' + echo ' pip install yapf' + exit 2 +fi +######### END PIP VERSION ########## + +########## PIPENV VERSION ########## +# if [ -z $(pipenv run which yapf) ]; then +# echo 'yapf not on path; can not format. Please install yapf:' +# echo ' pipenv install yapf' +# exit 2 +# fi +###### END PIPENV VERSION ########## + + +# Check for unstaged changes to files in the index. +CHANGED_FILES=(`git diff --name-only ${PYTHON_FILES[@]}`) +if [ "$CHANGED_FILES" ]; then + echo 'You have unstaged changes to some files in your commit; skipping ' + echo 'auto-format. Please stage, stash, or revert these changes. You may ' + echo 'find `git stash -k` helpful here.' + echo + echo 'Files with unstaged changes:' + for file in ${CHANGED_FILES[@]}; do + echo " $file" + done + exit 1 +fi + +# Format all staged files, then exit with an error code if any have uncommitted +# changes. +echo 'Formatting staged Python files . . .' + +########## PIP VERSION ############# +yapf -i -r ${PYTHON_FILES[@]} +######### END PIP VERSION ########## + +########## PIPENV VERSION ########## +# pipenv run yapf -i -r ${PYTHON_FILES[@]} +###### END PIPENV VERSION ########## + + +CHANGED_FILES=(`git diff --name-only ${PYTHON_FILES[@]}`) +if [ "$CHANGED_FILES" ]; then + echo 'Reformatted staged files. Please review and stage the changes.' + echo + echo 'Files updated:' + for file in ${CHANGED_FILES[@]}; do + echo " $file" + done + exit 1 +else + exit 0 +fi diff --git a/plugins/vim/autoload/yapf.vim b/plugins/vim/autoload/yapf.vim new file mode 100644 index 0000000..ad3e7fc --- /dev/null +++ b/plugins/vim/autoload/yapf.vim @@ -0,0 +1,49 @@ +" Copyright 2015 Google Inc. All Rights Reserved. +" +" Licensed under the Apache License, Version 2.0 (the "License"); +" you may not use this file except in compliance with the License. +" You may obtain a copy of the License at +" +" http://www.apache.org/licenses/LICENSE-2.0 +" +" Unless required by applicable law or agreed to in writing, software +" distributed under the License is distributed on an "AS IS" BASIS, +" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +" See the License for the specific language governing permissions and +" limitations under the License. +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +" VIM Autoload script for YAPF support +" +" Place this script in your ~/.vim/autoload directory. You can add accessors to +" ~/.vimrc, e.g.: +" +" map <C-Y> :call yapf#YAPF()<cr> +" imap <C-Y> <c-o>:call yapf#YAPF()<cr> +" +function! yapf#YAPF() range + " Determine range to format. + let l:line_ranges = a:firstline . '-' . a:lastline + let l:cmd = 'yapf --lines=' . l:line_ranges + + " Call YAPF with the current buffer + if exists('*systemlist') + let l:formatted_text = systemlist(l:cmd, join(getline(1, '$'), "\n") . "\n") + else + let l:formatted_text = + \ split(system(l:cmd, join(getline(1, '$'), "\n") . "\n"), "\n") + endif + + if v:shell_error + echohl ErrorMsg + echomsg printf('"%s" returned error: %s', l:cmd, l:formatted_text[-1]) + echohl None + return + endif + + " Update the buffer. + execute '1,' . string(line('$')) . 'delete' + call setline(1, l:formatted_text) + + " Reset cursor to first line of the formatted range. + call cursor(a:firstline, 1) +endfunction diff --git a/plugins/vim/plugin/yapf.vim b/plugins/vim/plugin/yapf.vim new file mode 100644 index 0000000..e2cc7fd --- /dev/null +++ b/plugins/vim/plugin/yapf.vim @@ -0,0 +1,25 @@ +" Copyright 2015 Google Inc. All Rights Reserved. +" +" Licensed under the Apache License, Version 2.0 (the "License"); +" you may not use this file except in compliance with the License. +" You may obtain a copy of the License at +" +" http://www.apache.org/licenses/LICENSE-2.0 +" +" Unless required by applicable law or agreed to in writing, software +" distributed under the License is distributed on an "AS IS" BASIS, +" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +" See the License for the specific language governing permissions and +" limitations under the License. +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +" VIM command for YAPF support +" +" Place this script in your ~/.vim/plugin directory. You can call the +" command YAPF. If you omit the range, it will reformat the whole +" buffer. +" +" example: +" :YAPF " formats whole buffer +" :'<,'>YAPF " formats lines selected in visual mode + +command! -range=% YAPF <line1>,<line2>call yapf#YAPF() diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..9e8a554 --- /dev/null +++ b/pylintrc @@ -0,0 +1,444 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable= + # disabled by me, + locally-disabled, + missing-docstring, + fixme, + # disabled by default, + import-star-module-level, + old-octal-literal, + oct-method, + print-statement, + unpacking-in-except, + parameter-unpacking, + backtick, + old-raise-syntax, + old-ne-operator, + long-suffix, + dict-view-method, + dict-iter-method, + metaclass-assignment, + next-method-called, + raising-string, + indexing-exception, + raw_input-builtin, + long-builtin, + file-builtin, + execfile-builtin, + coerce-builtin, + cmp-builtin, + buffer-builtin, + basestring-builtin, + apply-builtin, + filter-builtin-not-iterating, + using-cmp-argument, + useless-suppression, + range-builtin-not-iterating, + suppressed-message, + no-absolute-import, + old-division, + cmp-method, + reload-builtin, + zip-builtin-not-iterating, + intern-builtin, + unichr-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + input-builtin, + round-builtin, + hex-method, + nonzero-method, + map-builtin-not-iterating, + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,e,s,_,fd,fp + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +# original: +#function-rgx=[a-z_][a-z0-9_]{2,30}$ +function-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,40}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +# original: +#const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ +const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +# original: +#class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,40}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +# original: +#class-rgx=[A-Z_][a-zA-Z0-9]+$ +class-rgx=[a-zA-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +# original: +#method-rgx=[a-z_][a-z0-9_]{2,30}$ +method-rgx=[a-zA-Z_][a-zA-Z0-9_]{2,40}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). This supports can work +# with qualified names. +ignored-classes= + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2a9acf1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..21e9470 --- /dev/null +++ b/setup.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import codecs +import sys +import unittest + +from setuptools import setup, Command + +import yapf + + +class RunTests(Command): + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + loader = unittest.TestLoader() + tests = loader.discover('yapftests', pattern='*_test.py', top_level_dir='.') + runner = unittest.TextTestRunner() + results = runner.run(tests) + sys.exit(0 if results.wasSuccessful() else 1) + + +with codecs.open('README.rst', 'r', 'utf-8') as fd: + setup( + name='yapf', + version=yapf.__version__, + description='A formatter for Python code.', + long_description=fd.read(), + license='Apache License, Version 2.0', + author='Google Inc.', + maintainer='Bill Wendling', + maintainer_email='morbo@google.com', + packages=['yapf', 'yapf.yapflib', 'yapftests'], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Topic :: Software Development :: Quality Assurance', + ], + entry_points={ + 'console_scripts': ['yapf = yapf:run_main'], + }, + cmdclass={ + 'test': RunTests, + }, + ) @@ -0,0 +1,6 @@ +[tox] +envlist=py27,py34,py35,py36 + +[testenv] +commands= + python setup.py test diff --git a/yapf/__init__.py b/yapf/__init__.py new file mode 100644 index 0000000..22ab014 --- /dev/null +++ b/yapf/__init__.py @@ -0,0 +1,321 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""YAPF. + +YAPF uses the algorithm in clang-format to figure out the "best" formatting for +Python code. It looks at the program as a series of "unwrappable lines" --- +i.e., lines which, if there were no column limit, we would place all tokens on +that line. It then uses a priority queue to figure out what the best formatting +is --- i.e., the formatting with the least penalty. + +It differs from tools like autopep8 and pep8ify in that it doesn't just look for +violations of the style guide, but looks at the module as a whole, making +formatting decisions based on what's the best format for each line. + +If no filenames are specified, YAPF reads the code from stdin. +""" +from __future__ import print_function + +import argparse +import logging +import os +import sys + +from yapf.yapflib import errors +from yapf.yapflib import file_resources +from yapf.yapflib import py3compat +from yapf.yapflib import style +from yapf.yapflib import yapf_api + +__version__ = '0.22.0' + + +def main(argv): + """Main program. + + Arguments: + argv: command-line arguments, such as sys.argv (including the program name + in argv[0]). + + Returns: + Zero on successful program termination, non-zero otherwise. + With --diff: zero if there were no changes, non-zero otherwise. + + Raises: + YapfError: if none of the supplied files were Python files. + """ + parser = argparse.ArgumentParser(description='Formatter for Python code.') + parser.add_argument( + '-v', + '--version', + action='store_true', + help='show version number and exit') + + diff_inplace_group = parser.add_mutually_exclusive_group() + diff_inplace_group.add_argument( + '-d', + '--diff', + action='store_true', + help='print the diff for the fixed source') + diff_inplace_group.add_argument( + '-i', + '--in-place', + action='store_true', + help='make changes to files in place') + + lines_recursive_group = parser.add_mutually_exclusive_group() + lines_recursive_group.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + lines_recursive_group.add_argument( + '-l', + '--lines', + metavar='START-END', + action='append', + default=None, + help='range of lines to reformat, one-based') + + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=None, + help='patterns for files to exclude from formatting') + parser.add_argument( + '--style', + action='store', + help=('specify formatting style: either a style name (for example "pep8" ' + 'or "google"), or the name of a file with style settings. The ' + 'default is pep8 unless a %s or %s file located in the same ' + 'directory as the source or one of its parent directories ' + '(for stdin, the current directory is used).' % + (style.LOCAL_STYLE, style.SETUP_CONFIG))) + parser.add_argument( + '--style-help', + action='store_true', + help=('show style settings and exit; this output can be ' + 'saved to .style.yapf to make your settings ' + 'permanent')) + parser.add_argument( + '--no-local-style', + action='store_true', + help="don't search for local style definition") + parser.add_argument('--verify', action='store_true', help=argparse.SUPPRESS) + parser.add_argument( + '-p', + '--parallel', + action='store_true', + help=('Run yapf in parallel when formatting multiple files. Requires ' + 'concurrent.futures in Python 2.X')) + parser.add_argument( + '-vv', + '--verbose', + action='store_true', + help='Print out file names while processing') + + parser.add_argument( + 'files', nargs='*', help='Reads from stdin when no files are specified.') + args = parser.parse_args(argv[1:]) + + if args.version: + print('yapf {}'.format(__version__)) + return 0 + + style_config = args.style + + if args.style_help: + if style_config is None and not args.no_local_style: + style_config = file_resources.GetDefaultStyleForDir(os.getcwd()) + style.SetGlobalStyle(style.CreateStyleFromConfig(style_config)) + print('[style]') + for option, docstring in sorted(style.Help().items()): + for line in docstring.splitlines(): + print('#', line and ' ' or '', line, sep='') + print(option.lower(), '=', style.Get(option), sep='') + print() + return 0 + + if args.lines and len(args.files) > 1: + parser.error('cannot use -l/--lines with more than one file') + + lines = _GetLines(args.lines) if args.lines is not None else None + if not args.files: + # No arguments specified. Read code from stdin. + if args.in_place or args.diff: + parser.error('cannot use --in-place or --diff flags when reading ' + 'from stdin') + + original_source = [] + while True: + if sys.stdin.closed: + break + try: + # Use 'raw_input' instead of 'sys.stdin.read', because otherwise the + # user will need to hit 'Ctrl-D' more than once if they're inputting + # the program by hand. 'raw_input' throws an EOFError exception if + # 'Ctrl-D' is pressed, which makes it easy to bail out of this loop. + original_source.append(py3compat.raw_input()) + except EOFError: + break + + if style_config is None and not args.no_local_style: + style_config = file_resources.GetDefaultStyleForDir(os.getcwd()) + + source = [line.rstrip() for line in original_source] + reformatted_source, _ = yapf_api.FormatCode( + py3compat.unicode('\n'.join(source) + '\n'), + filename='<stdin>', + style_config=style_config, + lines=lines, + verify=args.verify) + file_resources.WriteReformattedCode('<stdout>', reformatted_source) + return 0 + + files = file_resources.GetCommandLineFiles(args.files, args.recursive, + args.exclude) + if not files: + raise errors.YapfError('Input filenames did not match any python files') + + changed = FormatFiles( + files, + lines, + style_config=args.style, + no_local_style=args.no_local_style, + in_place=args.in_place, + print_diff=args.diff, + verify=args.verify, + parallel=args.parallel, + verbose=args.verbose) + return 1 if changed and args.diff else 0 + + +def FormatFiles(filenames, + lines, + style_config=None, + no_local_style=False, + in_place=False, + print_diff=False, + verify=False, + parallel=False, + verbose=False): + """Format a list of files. + + Arguments: + filenames: (list of unicode) A list of files to reformat. + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. This argument + overrides the 'args.lines'. It can be used by third-party code (e.g., + IDEs) when reformatting a snippet of code. + style_config: (string) Style name or file path. + no_local_style: (string) If style_config is None don't search for + directory-local style configuration. + in_place: (bool) Modify the files in place. + print_diff: (bool) Instead of returning the reformatted source, return a + diff that turns the formatted source into reformatter source. + verify: (bool) True if reformatted code should be verified for syntax. + parallel: (bool) True if should format multiple files in parallel. + verbose: (bool) True if should print out filenames while processing. + + Returns: + True if the source code changed in any of the files being formatted. + """ + changed = False + if parallel: + import multiprocessing # pylint: disable=g-import-not-at-top + import concurrent.futures # pylint: disable=g-import-not-at-top + workers = min(multiprocessing.cpu_count(), len(filenames)) + with concurrent.futures.ProcessPoolExecutor(workers) as executor: + future_formats = [ + executor.submit(_FormatFile, filename, lines, style_config, + no_local_style, in_place, print_diff, verify, verbose) + for filename in filenames + ] + for future in concurrent.futures.as_completed(future_formats): + changed |= future.result() + else: + for filename in filenames: + changed |= _FormatFile(filename, lines, style_config, no_local_style, + in_place, print_diff, verify, verbose) + return changed + + +def _FormatFile(filename, + lines, + style_config=None, + no_local_style=False, + in_place=False, + print_diff=False, + verify=False, + verbose=False): + if verbose: + print('Reformatting %s' % filename) + if style_config is None and not no_local_style: + style_config = file_resources.GetDefaultStyleForDir( + os.path.dirname(filename)) + try: + reformatted_code, encoding, has_change = yapf_api.FormatFile( + filename, + in_place=in_place, + style_config=style_config, + lines=lines, + print_diff=print_diff, + verify=verify, + logger=logging.warning) + if not in_place and reformatted_code: + file_resources.WriteReformattedCode(filename, reformatted_code, encoding, + in_place) + return has_change + except SyntaxError as e: + e.filename = filename + raise + + +def _GetLines(line_strings): + """Parses the start and end lines from a line string like 'start-end'. + + Arguments: + line_strings: (array of string) A list of strings representing a line + range like 'start-end'. + + Returns: + A list of tuples of the start and end line numbers. + + Raises: + ValueError: If the line string failed to parse or was an invalid line range. + """ + lines = [] + for line_string in line_strings: + # The 'list' here is needed by Python 3. + line = list(map(int, line_string.split('-', 1))) + if line[0] < 1: + raise errors.YapfError('invalid start of line range: %r' % line) + if line[0] > line[1]: + raise errors.YapfError('end comes before start in line range: %r', line) + lines.append(tuple(line)) + return lines + + +def run_main(): # pylint: disable=invalid-name + try: + sys.exit(main(sys.argv)) + except errors.YapfError as e: + sys.stderr.write('yapf: ' + str(e) + '\n') + sys.exit(1) + + +if __name__ == '__main__': + run_main() diff --git a/yapf/__main__.py b/yapf/__main__.py new file mode 100644 index 0000000..f1d0e32 --- /dev/null +++ b/yapf/__main__.py @@ -0,0 +1,18 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Main entry point.""" +# pylint: disable=invalid-name +import yapf + +yapf.run_main() diff --git a/yapf/yapflib/__init__.py b/yapf/yapflib/__init__.py new file mode 100644 index 0000000..e7522b2 --- /dev/null +++ b/yapf/yapflib/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/yapf/yapflib/blank_line_calculator.py b/yapf/yapflib/blank_line_calculator.py new file mode 100644 index 0000000..c239ee7 --- /dev/null +++ b/yapf/yapflib/blank_line_calculator.py @@ -0,0 +1,176 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Calculate the number of blank lines between top-level entities. + +Calculates how many blank lines we need between classes, functions, and other +entities at the same level. + + CalculateBlankLines(): the main function exported by this module. + +Annotations: + newlines: The number of newlines required before the node. +""" + +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import style + +_NO_BLANK_LINES = 1 +_ONE_BLANK_LINE = 2 +_TWO_BLANK_LINES = 3 + +_PYTHON_STATEMENTS = frozenset({ + 'small_stmt', 'expr_stmt', 'print_stmt', 'del_stmt', 'pass_stmt', + 'break_stmt', 'continue_stmt', 'return_stmt', 'raise_stmt', 'yield_stmt', + 'import_stmt', 'global_stmt', 'exec_stmt', 'assert_stmt', 'if_stmt', + 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt', 'nonlocal_stmt', + 'async_stmt', 'simple_stmt' +}) + + +def CalculateBlankLines(tree): + """Run the blank line calculator visitor over the tree. + + This modifies the tree in place. + + Arguments: + tree: the top-level pytree node to annotate with subtypes. + """ + blank_line_calculator = _BlankLineCalculator() + blank_line_calculator.Visit(tree) + + +class _BlankLineCalculator(pytree_visitor.PyTreeVisitor): + """_BlankLineCalculator - see file-level docstring for a description.""" + + def __init__(self): + self.class_level = 0 + self.function_level = 0 + self.last_comment_lineno = 0 + self.last_was_decorator = False + self.last_was_class_or_function = False + + def Visit_simple_stmt(self, node): # pylint: disable=invalid-name + self.DefaultNodeVisit(node) + if pytree_utils.NodeName(node.children[0]) == 'COMMENT': + self.last_comment_lineno = node.children[0].lineno + + def Visit_decorator(self, node): # pylint: disable=invalid-name + if (self.last_comment_lineno and + self.last_comment_lineno == node.children[0].lineno - 1): + self._SetNumNewlines(node.children[0], _NO_BLANK_LINES) + else: + self._SetNumNewlines(node.children[0], self._GetNumNewlines(node)) + for child in node.children: + self.Visit(child) + self.last_was_decorator = True + + def Visit_classdef(self, node): # pylint: disable=invalid-name + self.last_was_class_or_function = False + index = self._SetBlankLinesBetweenCommentAndClassFunc(node) + self.last_was_decorator = False + self.class_level += 1 + for child in node.children[index:]: + self.Visit(child) + self.class_level -= 1 + self.last_was_class_or_function = True + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + self.last_was_class_or_function = False + index = self._SetBlankLinesBetweenCommentAndClassFunc(node) + if _AsyncFunction(node): + index = self._SetBlankLinesBetweenCommentAndClassFunc( + node.prev_sibling.parent) + self._SetNumNewlines(node.children[0], None) + else: + index = self._SetBlankLinesBetweenCommentAndClassFunc(node) + self.last_was_decorator = False + self.function_level += 1 + for child in node.children[index:]: + self.Visit(child) + self.function_level -= 1 + self.last_was_class_or_function = True + + def DefaultNodeVisit(self, node): + """Override the default visitor for Node. + + This will set the blank lines required if the last entity was a class or + function. + + Arguments: + node: (pytree.Node) The node to visit. + """ + if self.last_was_class_or_function: + if pytree_utils.NodeName(node) in _PYTHON_STATEMENTS: + leaf = pytree_utils.FirstLeafNode(node) + self._SetNumNewlines(leaf, self._GetNumNewlines(leaf)) + self.last_was_class_or_function = False + super(_BlankLineCalculator, self).DefaultNodeVisit(node) + + def _SetBlankLinesBetweenCommentAndClassFunc(self, node): + """Set the number of blanks between a comment and class or func definition. + + Class and function definitions have leading comments as children of the + classdef and functdef nodes. + + Arguments: + node: (pytree.Node) The classdef or funcdef node. + + Returns: + The index of the first child past the comment nodes. + """ + index = 0 + while pytree_utils.IsCommentStatement(node.children[index]): + # Standalone comments are wrapped in a simple_stmt node with the comment + # node as its only child. + self.Visit(node.children[index].children[0]) + if not self.last_was_decorator: + self._SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE) + index += 1 + if (index and node.children[index].lineno - + 1 == node.children[index - 1].children[0].lineno): + self._SetNumNewlines(node.children[index], _NO_BLANK_LINES) + else: + if self.last_comment_lineno + 1 == node.children[index].lineno: + num_newlines = _NO_BLANK_LINES + else: + num_newlines = self._GetNumNewlines(node) + self._SetNumNewlines(node.children[index], num_newlines) + return index + + def _GetNumNewlines(self, node): + if self.last_was_decorator: + return _NO_BLANK_LINES + elif self._IsTopLevel(node): + return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION') + return _ONE_BLANK_LINE + + def _SetNumNewlines(self, node, num_newlines): + pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES, + num_newlines) + + def _IsTopLevel(self, node): + return (not (self.class_level or self.function_level) and + _StartsInZerothColumn(node)) + + +def _StartsInZerothColumn(node): + return (pytree_utils.FirstLeafNode(node).column == 0 or + (_AsyncFunction(node) and node.prev_sibling.column == 0)) + + +def _AsyncFunction(node): + return (py3compat.PY3 and node.prev_sibling and + pytree_utils.NodeName(node.prev_sibling) == 'ASYNC') diff --git a/yapf/yapflib/comment_splicer.py b/yapf/yapflib/comment_splicer.py new file mode 100644 index 0000000..af999d2 --- /dev/null +++ b/yapf/yapflib/comment_splicer.py @@ -0,0 +1,354 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Comment splicer for lib2to3 trees. + +The lib2to3 syntax tree produced by the parser holds comments and whitespace in +prefix attributes of nodes, rather than nodes themselves. This module provides +functionality to splice comments out of prefixes and into nodes of their own, +making them easier to process. + + SpliceComments(): the main function exported by this module. +""" + +from lib2to3 import pygram +from lib2to3 import pytree +from lib2to3.pgen2 import token + +from yapf.yapflib import pytree_utils + + +def SpliceComments(tree): + """Given a pytree, splice comments into nodes of their own right. + + Extract comments from the prefixes where they are housed after parsing. + The prefixes that previously housed the comments become empty. + + Args: + tree: a pytree.Node - the tree to work on. The tree is modified by this + function. + """ + # The previous leaf node encountered in the traversal. + # This is a list because Python 2.x doesn't have 'nonlocal' :) + prev_leaf = [None] + _AnnotateIndents(tree) + + def _VisitNodeRec(node): + # This loop may insert into node.children, so we'll iterate over a copy. + for child in node.children[:]: + if isinstance(child, pytree.Node): + # Nodes don't have prefixes. + _VisitNodeRec(child) + else: + if child.prefix.lstrip().startswith('#'): + # We have a comment prefix in this child, so splicing is needed. + comment_prefix = child.prefix + comment_lineno = child.lineno - comment_prefix.count('\n') + comment_column = child.column + + # Remember the leading indentation of this prefix and clear it. + # Mopping up the prefix is important because we may go over this same + # child in the next iteration... + child_prefix = child.prefix.lstrip('\n') + prefix_indent = child_prefix[:child_prefix.find('#')] + if '\n' in prefix_indent: + prefix_indent = prefix_indent[prefix_indent.rfind('\n') + 1:] + child.prefix = '' + + if child.type == token.NEWLINE: + # If the prefix was on a NEWLINE leaf, it's part of the line so it + # will be inserted after the previously encountered leaf. + # We can't just insert it before the NEWLINE node, because as a + # result of the way pytrees are organized, this node can be under + # an inappropriate parent. + comment_column -= len(comment_prefix.lstrip()) + pytree_utils.InsertNodesAfter( + _CreateCommentsFromPrefix( + comment_prefix, + comment_lineno, + comment_column, + standalone=False), prev_leaf[0]) + elif child.type == token.DEDENT: + # Comment prefixes on DEDENT nodes also deserve special treatment, + # because their final placement depends on their prefix. + # We'll look for an ancestor of this child with a matching + # indentation, and insert the comment before it if the ancestor is + # on a DEDENT node and after it otherwise. + # + # lib2to3 places comments that should be separated into the same + # DEDENT node. For example, "comment 1" and "comment 2" will be + # combined. + # + # def _(): + # for x in y: + # pass + # # comment 1 + # + # # comment 2 + # pass + # + # In this case, we need to split them up ourselves. + + # Split into groups of comments at decreasing levels of indentation + comment_groups = [] + comment_column = None + for cmt in comment_prefix.split('\n'): + col = cmt.find('#') + if col < 0: + if comment_column is None: + # Skip empty lines at the top of the first comment group + comment_lineno += 1 + continue + elif comment_column is None or col < comment_column: + comment_column = col + comment_indent = cmt[:comment_column] + comment_groups.append((comment_column, comment_indent, [])) + comment_groups[-1][-1].append(cmt) + + # Insert a node for each group + for comment_column, comment_indent, comment_group in comment_groups: + ancestor_at_indent = _FindAncestorAtIndent(child, comment_indent) + if ancestor_at_indent.type == token.DEDENT: + InsertNodes = pytree_utils.InsertNodesBefore # pylint: disable=invalid-name + else: + InsertNodes = pytree_utils.InsertNodesAfter # pylint: disable=invalid-name + InsertNodes( + _CreateCommentsFromPrefix( + '\n'.join(comment_group) + '\n', + comment_lineno, + comment_column, + standalone=True), ancestor_at_indent) + comment_lineno += len(comment_group) + else: + # Otherwise there are two cases. + # + # 1. The comment is on its own line + # 2. The comment is part of an expression. + # + # Unfortunately, it's fairly difficult to distinguish between the + # two in lib2to3 trees. The algorithm here is to determine whether + # child is the first leaf in the statement it belongs to. If it is, + # then the comment (which is a prefix) belongs on a separate line. + # If it is not, it means the comment is buried deep in the statement + # and is part of some expression. + stmt_parent = _FindStmtParent(child) + + for leaf_in_parent in stmt_parent.leaves(): + if leaf_in_parent.type == token.NEWLINE: + continue + elif id(leaf_in_parent) == id(child): + # This comment stands on its own line, and it has to be inserted + # into the appropriate parent. We'll have to find a suitable + # parent to insert into. See comments above + # _STANDALONE_LINE_NODES for more details. + node_with_line_parent = _FindNodeWithStandaloneLineParent(child) + pytree_utils.InsertNodesBefore( + _CreateCommentsFromPrefix( + comment_prefix, comment_lineno, 0, standalone=True), + node_with_line_parent) + break + else: + if comment_lineno == prev_leaf[0].lineno: + comment_lines = comment_prefix.splitlines() + value = comment_lines[0].lstrip() + if value.rstrip('\n'): + comment_column = prev_leaf[0].column + comment_column += len(prev_leaf[0].value) + comment_column += ( + len(comment_lines[0]) - len(comment_lines[0].lstrip())) + comment_leaf = pytree.Leaf( + type=token.COMMENT, + value=value.rstrip('\n'), + context=('', (comment_lineno, comment_column))) + pytree_utils.InsertNodesAfter([comment_leaf], prev_leaf[0]) + comment_prefix = '\n'.join(comment_lines[1:]) + comment_lineno += 1 + + rindex = (0 if '\n' not in comment_prefix.rstrip() else + comment_prefix.rstrip().rindex('\n') + 1) + comment_column = ( + len(comment_prefix[rindex:]) - len( + comment_prefix[rindex:].lstrip())) + comments = _CreateCommentsFromPrefix( + comment_prefix, + comment_lineno, + comment_column, + standalone=False) + pytree_utils.InsertNodesBefore(comments, child) + break + + prev_leaf[0] = child + + _VisitNodeRec(tree) + + +def _CreateCommentsFromPrefix(comment_prefix, + comment_lineno, + comment_column, + standalone=False): + """Create pytree nodes to represent the given comment prefix. + + Args: + comment_prefix: (unicode) the text of the comment from the node's prefix. + comment_lineno: (int) the line number for the start of the comment. + comment_column: (int) the column for the start of the comment. + standalone: (bool) determines if the comment is standalone or not. + + Returns: + The simple_stmt nodes if this is a standalone comment, otherwise a list of + new COMMENT leafs. The prefix may consist of multiple comment blocks, + separated by blank lines. Each block gets its own leaf. + """ + # The comment is stored in the prefix attribute, with no lineno of its + # own. So we only know at which line it ends. To find out at which line it + # starts, look at how many newlines the comment itself contains. + comments = [] + + lines = comment_prefix.split('\n') + index = 0 + while index < len(lines): + comment_block = [] + while index < len(lines) and lines[index].lstrip().startswith('#'): + comment_block.append(lines[index].strip()) + index += 1 + + if comment_block: + new_lineno = comment_lineno + index - 1 + comment_block[0] = comment_block[0].strip() + comment_block[-1] = comment_block[-1].strip() + comment_leaf = pytree.Leaf( + type=token.COMMENT, + value='\n'.join(comment_block), + context=('', (new_lineno, comment_column))) + comment_node = comment_leaf if not standalone else pytree.Node( + pygram.python_symbols.simple_stmt, [comment_leaf]) + comments.append(comment_node) + + while index < len(lines) and not lines[index].lstrip(): + index += 1 + + return comments + + +# "Standalone line nodes" are tree nodes that have to start a new line in Python +# code (and cannot follow a ';' or ':'). Other nodes, like 'expr_stmt', serve as +# parents of other nodes but can come later in a line. This is a list of +# standalone line nodes in the grammar. It is meant to be exhaustive +# *eventually*, and we'll modify it with time as we discover more corner cases +# in the parse tree. +# +# When splicing a standalone comment (i.e. a comment that appears on its own +# line, not on the same line with other code), it's important to insert it into +# an appropriate parent of the node it's attached to. An appropriate parent +# is the first "standaline line node" in the parent chain of a node. +_STANDALONE_LINE_NODES = frozenset([ + 'suite', 'if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt', + 'funcdef', 'classdef', 'decorated', 'file_input' +]) + + +def _FindNodeWithStandaloneLineParent(node): + """Find a node whose parent is a 'standalone line' node. + + See the comment above _STANDALONE_LINE_NODES for more details. + + Arguments: + node: node to start from + + Returns: + Suitable node that's either the node itself or one of its ancestors. + """ + if pytree_utils.NodeName(node.parent) in _STANDALONE_LINE_NODES: + return node + else: + # This is guaranteed to terminate because 'file_input' is the root node of + # any pytree. + return _FindNodeWithStandaloneLineParent(node.parent) + + +# "Statement nodes" are standalone statements. The don't have to start a new +# line. +_STATEMENT_NODES = frozenset(['simple_stmt']) | _STANDALONE_LINE_NODES + + +def _FindStmtParent(node): + """Find the nearest parent of node that is a statement node. + + Arguments: + node: node to start from + + Returns: + Nearest parent (or node itself, if suitable). + """ + if pytree_utils.NodeName(node) in _STATEMENT_NODES: + return node + else: + return _FindStmtParent(node.parent) + + +def _FindAncestorAtIndent(node, indent): + """Find an ancestor of node with the given indentation. + + Arguments: + node: node to start from. This must not be the tree root. + indent: indentation string for the ancestor we're looking for. + See _AnnotateIndents for more details. + + Returns: + An ancestor node with suitable indentation. If no suitable ancestor is + found, the closest ancestor to the tree root is returned. + """ + if node.parent.parent is None: + # Our parent is the tree root, so there's nowhere else to go. + return node + + # If the parent has an indent annotation, and it's shorter than node's + # indent, this is a suitable ancestor. + # The reason for "shorter" rather than "equal" is that comments may be + # improperly indented (i.e. by three spaces, where surrounding statements + # have either zero or two or four), and we don't want to propagate them all + # the way to the root. + parent_indent = pytree_utils.GetNodeAnnotation( + node.parent, pytree_utils.Annotation.CHILD_INDENT) + if parent_indent is not None and indent.startswith(parent_indent): + return node + else: + # Keep looking up the tree. + return _FindAncestorAtIndent(node.parent, indent) + + +def _AnnotateIndents(tree): + """Annotate the tree with child_indent annotations. + + A child_indent annotation on a node specifies the indentation (as a string, + like " ") of its children. It is inferred from the INDENT child of a node. + + Arguments: + tree: root of a pytree. The pytree is modified to add annotations to nodes. + + Raises: + RuntimeError: if the tree is malformed. + """ + # Annotate the root of the tree with zero indent. + if tree.parent is None: + pytree_utils.SetNodeAnnotation(tree, pytree_utils.Annotation.CHILD_INDENT, + '') + for child in tree.children: + if child.type == token.INDENT: + child_indent = pytree_utils.GetNodeAnnotation( + tree, pytree_utils.Annotation.CHILD_INDENT) + if child_indent is not None and child_indent != child.value: + raise RuntimeError('inconsistent indentation for child', (tree, child)) + pytree_utils.SetNodeAnnotation(tree, pytree_utils.Annotation.CHILD_INDENT, + child.value) + _AnnotateIndents(child) diff --git a/yapf/yapflib/continuation_splicer.py b/yapf/yapflib/continuation_splicer.py new file mode 100644 index 0000000..b86188c --- /dev/null +++ b/yapf/yapflib/continuation_splicer.py @@ -0,0 +1,52 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Insert "continuation" nodes into lib2to3 tree. + +The "backslash-newline" continuation marker is shoved into the node's prefix. +Pull them out and make it into nodes of their own. + + SpliceContinuations(): the main function exported by this module. +""" + +from lib2to3 import pytree + +from yapf.yapflib import format_token + + +def SpliceContinuations(tree): + """Given a pytree, splice the continuation marker into nodes. + + Arguments: + tree: (pytree.Node) The tree to work on. The tree is modified by this + function. + """ + + def RecSplicer(node): + """Inserts a continuation marker into the node.""" + if isinstance(node, pytree.Leaf): + if node.prefix.lstrip().startswith('\\\n'): + new_lineno = node.lineno - node.prefix.count('\n') + return pytree.Leaf( + type=format_token.CONTINUATION, + value=node.prefix, + context=('', (new_lineno, 0))) + return None + num_inserted = 0 + for index, child in enumerate(node.children[:]): + continuation_node = RecSplicer(child) + if continuation_node: + node.children.insert(index + num_inserted, continuation_node) + num_inserted += 1 + + RecSplicer(tree) diff --git a/yapf/yapflib/errors.py b/yapf/yapflib/errors.py new file mode 100644 index 0000000..3946275 --- /dev/null +++ b/yapf/yapflib/errors.py @@ -0,0 +1,23 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""YAPF error object.""" + + +class YapfError(Exception): + """Parent class for user errors or input errors. + + Exceptions of this type are handled by the command line tool + and result in clear error messages, as opposed to backtraces. + """ + pass diff --git a/yapf/yapflib/file_resources.py b/yapf/yapflib/file_resources.py new file mode 100644 index 0000000..6e7202d --- /dev/null +++ b/yapf/yapflib/file_resources.py @@ -0,0 +1,191 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Interface to file resources. + +This module provides functions for interfacing with files: opening, writing, and +querying. +""" + +import fnmatch +import os +import re + +from lib2to3.pgen2 import tokenize + +from yapf.yapflib import errors +from yapf.yapflib import py3compat +from yapf.yapflib import style + +CR = '\r' +LF = '\n' +CRLF = '\r\n' + + +def GetDefaultStyleForDir(dirname): + """Return default style name for a given directory. + + Looks for .style.yapf or setup.cfg in the parent directories. + + Arguments: + dirname: (unicode) The name of the directory. + + Returns: + The filename if found, otherwise return the global default (pep8). + """ + dirname = os.path.abspath(dirname) + while True: + # See if we have a .style.yapf file. + style_file = os.path.join(dirname, style.LOCAL_STYLE) + if os.path.exists(style_file): + return style_file + + # See if we have a setup.cfg file with a '[yapf]' section. + config_file = os.path.join(dirname, style.SETUP_CONFIG) + if os.path.exists(config_file): + with open(config_file) as fd: + config = py3compat.ConfigParser() + config.read_file(fd) + if config.has_section('yapf'): + return config_file + + dirname = os.path.dirname(dirname) + if (not dirname or not os.path.basename(dirname) or + dirname == os.path.abspath(os.path.sep)): + break + + global_file = os.path.expanduser(style.GLOBAL_STYLE) + if os.path.exists(global_file): + return global_file + + return style.DEFAULT_STYLE + + +def GetCommandLineFiles(command_line_file_list, recursive, exclude): + """Return the list of files specified on the command line.""" + return _FindPythonFiles(command_line_file_list, recursive, exclude) + + +def WriteReformattedCode(filename, + reformatted_code, + encoding='', + in_place=False): + """Emit the reformatted code. + + Write the reformatted code into the file, if in_place is True. Otherwise, + write to stdout. + + Arguments: + filename: (unicode) The name of the unformatted file. + reformatted_code: (unicode) The reformatted code. + encoding: (unicode) The encoding of the file. + in_place: (bool) If True, then write the reformatted code to the file. + """ + if in_place: + with py3compat.open_with_encoding( + filename, mode='w', encoding=encoding, newline='') as fd: + fd.write(reformatted_code) + else: + py3compat.EncodeAndWriteToStdout(reformatted_code) + + +def LineEnding(lines): + """Retrieve the line ending of the original source.""" + endings = {CRLF: 0, CR: 0, LF: 0} + for line in lines: + if line.endswith(CRLF): + endings[CRLF] += 1 + elif line.endswith(CR): + endings[CR] += 1 + elif line.endswith(LF): + endings[LF] += 1 + return (sorted(endings, key=endings.get, reverse=True) or [LF])[0] + + +def _FindPythonFiles(filenames, recursive, exclude): + """Find all Python files.""" + if exclude and any(e.startswith('./') for e in exclude): + raise errors.YapfError("path in '--exclude' should not start with ./") + + python_files = [] + for filename in filenames: + if filename != '.' and exclude and IsIgnored(filename, exclude): + continue + if os.path.isdir(filename): + if recursive: + # TODO(morbo): Look into a version of os.walk that can handle recursion. + excluded_dirs = [] + for dirpath, _, filelist in os.walk(filename): + if dirpath != '.' and exclude and IsIgnored(dirpath, exclude): + excluded_dirs.append(dirpath) + continue + elif any(dirpath.startswith(e) for e in excluded_dirs): + continue + for f in filelist: + filepath = os.path.join(dirpath, f) + if exclude and IsIgnored(filepath, exclude): + continue + if IsPythonFile(filepath): + python_files.append(filepath) + else: + raise errors.YapfError( + "directory specified without '--recursive' flag: %s" % filename) + elif os.path.isfile(filename): + python_files.append(filename) + + return python_files + + +def IsIgnored(path, exclude): + """Return True if filename matches any patterns in exclude.""" + path = path.lstrip('/') + while path.startswith('./'): + path = path[2:] + return any(fnmatch.fnmatch(path, e.rstrip('/')) for e in exclude) + + +def IsPythonFile(filename): + """Return True if filename is a Python file.""" + if os.path.splitext(filename)[1] == '.py': + return True + + try: + with open(filename, 'rb') as fd: + encoding = tokenize.detect_encoding(fd.readline)[0] + + # Check for correctness of encoding. + with py3compat.open_with_encoding( + filename, mode='r', encoding=encoding) as fd: + fd.read() + except UnicodeDecodeError: + encoding = 'latin-1' + except (IOError, SyntaxError): + # If we fail to detect encoding (or the encoding cookie is incorrect - which + # will make detect_encoding raise SyntaxError), assume it's not a Python + # file. + return False + + try: + with py3compat.open_with_encoding( + filename, mode='r', encoding=encoding) as fd: + first_line = fd.readline(256) + except IOError: + return False + + return re.match(r'^#!.*\bpython[23]?\b', first_line) + + +def FileEncoding(filename): + """Return the file's encoding.""" + with open(filename, 'rb') as fd: + return tokenize.detect_encoding(fd.readline)[0] diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py new file mode 100644 index 0000000..bff8ea3 --- /dev/null +++ b/yapf/yapflib/format_decision_state.py @@ -0,0 +1,1011 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implements a format decision state object that manages whitespace decisions. + +Each token is processed one at a time, at which point its whitespace formatting +decisions are made. A graph of potential whitespace formattings is created, +where each node in the graph is a format decision state object. The heuristic +tries formatting the token with and without a newline before it to determine +which one has the least penalty. Therefore, the format decision state object for +each decision needs to be its own unique copy. + +Once the heuristic determines the best formatting, it makes a non-dry run pass +through the code to commit the whitespace formatting. + + FormatDecisionState: main class exported by this module. +""" + +from yapf.yapflib import format_token +from yapf.yapflib import object_state +from yapf.yapflib import split_penalty +from yapf.yapflib import style +from yapf.yapflib import unwrapped_line + + +class FormatDecisionState(object): + """The current state when indenting an unwrapped line. + + The FormatDecisionState object is meant to be copied instead of referenced. + + Attributes: + first_indent: The indent of the first token. + column: The number of used columns in the current line. + next_token: The next token to be formatted. + paren_level: The level of nesting inside (), [], and {}. + lowest_level_on_line: The lowest paren_level on the current line. + newline: Indicates if a newline is added along the edge to this format + decision state node. + previous: The previous format decision state in the decision tree. + stack: A stack (of _ParenState) keeping track of properties applying to + parenthesis levels. + comp_stack: A stack (of ComprehensionState) keeping track of properties + applying to comprehensions. + ignore_stack_for_comparison: Ignore the stack of _ParenState for state + comparison. + """ + + def __init__(self, line, first_indent): + """Initializer. + + Initializes to the state after placing the first token from 'line' at + 'first_indent'. + + Arguments: + line: (UnwrappedLine) The unwrapped line we're currently processing. + first_indent: (int) The indent of the first token. + """ + self.next_token = line.first + self.column = first_indent + self.line = line + self.paren_level = 0 + self.lowest_level_on_line = 0 + self.ignore_stack_for_comparison = False + self.stack = [_ParenState(first_indent, first_indent)] + self.comp_stack = [] + self.first_indent = first_indent + self.newline = False + self.previous = None + self.column_limit = style.Get('COLUMN_LIMIT') + + def Clone(self): + """Clones a FormatDecisionState object.""" + new = FormatDecisionState(self.line, self.first_indent) + new.next_token = self.next_token + new.column = self.column + new.line = self.line + new.paren_level = self.paren_level + new.line.depth = self.line.depth + new.lowest_level_on_line = self.lowest_level_on_line + new.ignore_stack_for_comparison = self.ignore_stack_for_comparison + new.first_indent = self.first_indent + new.newline = self.newline + new.previous = self.previous + new.stack = [state.Clone() for state in self.stack] + new.comp_stack = [state.Clone() for state in self.comp_stack] + return new + + def __eq__(self, other): + # Note: 'first_indent' is implicit in the stack. Also, we ignore 'previous', + # because it shouldn't have a bearing on this comparison. (I.e., it will + # report equal if 'next_token' does.) + return (self.next_token == other.next_token and + self.column == other.column and + self.paren_level == other.paren_level and + self.line.depth == other.line.depth and + self.lowest_level_on_line == other.lowest_level_on_line and + (self.ignore_stack_for_comparison or + other.ignore_stack_for_comparison or + self.stack == other.stack and self.comp_stack == other.comp_stack)) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.next_token, self.column, self.paren_level, + self.line.depth, self.lowest_level_on_line)) + + def __repr__(self): + return ('column::%d, next_token::%s, paren_level::%d, stack::[\n\t%s' % + (self.column, repr(self.next_token), self.paren_level, + '\n\t'.join(repr(s) for s in self.stack) + ']')) + + def CanSplit(self, must_split): + """Determine if we can split before the next token. + + Arguments: + must_split: (bool) A newline was required before this token. + + Returns: + True if the line can be split before the next token. + """ + current = self.next_token + previous = current.previous_token + + if current.is_pseudo_paren: + return False + + if (not must_split and + format_token.Subtype.DICTIONARY_KEY_PART in current.subtypes and + format_token.Subtype.DICTIONARY_KEY not in current.subtypes and + not style.Get('ALLOW_MULTILINE_DICTIONARY_KEYS')): + # In some situations, a dictionary may be multiline, but pylint doesn't + # like it. So don't allow it unless forced to. + return False + + if (not must_split and + format_token.Subtype.DICTIONARY_VALUE in current.subtypes and + not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')): + return False + + if previous and previous.value == '(' and current.value == ')': + # Don't split an empty function call list if we aren't splitting before + # dict values. + token = previous.previous_token + while token: + prev = token.previous_token + if not prev or prev.name not in {'NAME', 'DOT'}: + break + token = token.previous_token + if token and format_token.Subtype.DICTIONARY_VALUE in token.subtypes: + if not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE'): + return False + + if previous and previous.value == '.' and current.value == '.': + return False + + return current.can_break_before + + def MustSplit(self): + """Returns True if the line must split before the next token.""" + current = self.next_token + previous = current.previous_token + + if current.is_pseudo_paren: + return False + + if current.must_break_before: + return True + + if not previous: + return False + + if style.Get('SPLIT_ALL_COMMA_SEPARATED_VALUES') and previous.value == ',': + return True + + if (self.stack[-1].split_before_closing_bracket and + current.value in '}]' and style.Get('SPLIT_BEFORE_CLOSING_BRACKET')): + # Split before the closing bracket if we can. + return current.node_split_penalty != split_penalty.UNBREAKABLE + + if (current.value == ')' and previous.value == ',' and + not _IsSingleElementTuple(current.matching_bracket)): + return True + + # Prevent splitting before the first argument in compound statements + # with the exception of function declarations. + if (style.Get('SPLIT_BEFORE_FIRST_ARGUMENT') and + _IsCompoundStatement(self.line.first) and + not _IsFunctionDef(self.line.first)): + return False + + ########################################################################### + # List Splitting + if (style.Get('DEDENT_CLOSING_BRACKETS') or + style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')): + bracket = current if current.ClosesScope() else previous + if format_token.Subtype.SUBSCRIPT_BRACKET not in bracket.subtypes: + if bracket.OpensScope(): + if style.Get('COALESCE_BRACKETS'): + if current.OpensScope(): + # Prefer to keep all opening brackets together. + return False + + if (not _IsLastScopeInLine(bracket) or + unwrapped_line.IsSurroundedByBrackets(bracket)): + last_token = bracket.matching_bracket + else: + last_token = _LastTokenInLine(bracket.matching_bracket) + + if not self._FitsOnLine(bracket, last_token): + # Split before the first element if the whole list can't fit on a + # single line. + self.stack[-1].split_before_closing_bracket = True + return True + + elif style.Get('DEDENT_CLOSING_BRACKETS') and current.ClosesScope(): + # Split before and dedent the closing bracket. + return self.stack[-1].split_before_closing_bracket + + if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and + current.is_name): + # An expression that's surrounded by parens gets split after the opening + # parenthesis. + def SurroundedByParens(token): + """Check if it's an expression surrounded by parentheses.""" + while token: + if token.value == ',': + return False + if token.value == ')': + return not token.next_token + if token.OpensScope(): + token = token.matching_bracket.next_token + else: + token = token.next_token + return False + + if (previous.value == '(' and not previous.is_pseudo_paren and + not unwrapped_line.IsSurroundedByBrackets(previous)): + pptoken = previous.previous_token + if (pptoken and not pptoken.is_name and not pptoken.is_keyword and + SurroundedByParens(current)): + return True + + if (current.is_name or current.is_string) and previous.value == ',': + # If the list has function calls in it and the full list itself cannot + # fit on the line, then we want to split. Otherwise, we'll get something + # like this: + # + # X = [ + # Bar(xxx='some string', + # yyy='another long string', + # zzz='a third long string'), Bar( + # xxx='some string', + # yyy='another long string', + # zzz='a third long string') + # ] + # + # or when a string formatting syntax. + func_call_or_string_format = False + tok = current.next_token + if current.is_name: + while tok and (tok.is_name or tok.value == '.'): + tok = tok.next_token + func_call_or_string_format = tok and tok.value == '(' + elif current.is_string: + while tok and tok.is_string: + tok = tok.next_token + func_call_or_string_format = tok and tok.value == '%' + if func_call_or_string_format: + open_bracket = unwrapped_line.IsSurroundedByBrackets(current) + if open_bracket: + if open_bracket.value in '[{': + if not self._FitsOnLine(open_bracket, + open_bracket.matching_bracket): + return True + elif tok.value == '(': + if not self._FitsOnLine(current, tok.matching_bracket): + return True + + ########################################################################### + # Dict/Set Splitting + if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE') and + format_token.Subtype.DICTIONARY_KEY in current.subtypes and + not current.is_comment): + # Place each dictionary entry onto its own line. + if previous.value == '{' and previous.previous_token: + opening = _GetOpeningBracket(previous.previous_token) + if (opening and opening.value == '(' and opening.previous_token and + opening.previous_token.is_name): + # This is a dictionary that's an argument to a function. + if (self._FitsOnLine(previous, previous.matching_bracket) and + previous.matching_bracket.next_token and + (not opening.matching_bracket.next_token or + opening.matching_bracket.next_token.value != '.') and + _ScopeHasNoCommas(previous)): + # Don't split before the key if: + # - The dictionary fits on a line, and + # - The function call isn't part of a builder-style call and + # - The dictionary has one entry and no trailing comma + return False + return True + + if (style.Get('SPLIT_BEFORE_DICT_SET_GENERATOR') and + format_token.Subtype.DICT_SET_GENERATOR in current.subtypes): + # Split before a dict/set generator. + return True + + if (format_token.Subtype.DICTIONARY_VALUE in current.subtypes or + (previous.is_pseudo_paren and previous.value == '(' and + not current.is_comment)): + # Split before the dictionary value if we can't fit every dictionary + # entry on its own line. + if not current.OpensScope(): + opening = _GetOpeningBracket(current) + if not self._EachDictEntryFitsOnOneLine(opening): + return style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE') + + if previous.value == '{': + # Split if the dict/set cannot fit on one line and ends in a comma. + closing = previous.matching_bracket + if (not self._FitsOnLine(previous, closing) and + closing.previous_token.value == ','): + self.stack[-1].split_before_closing_bracket = True + return True + + ########################################################################### + # Argument List Splitting + if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and not current.is_comment and + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in + current.subtypes): + if (previous.value not in {'=', ':', '*', '**'} and + current.value not in ':=,)' and not _IsFunctionDefinition(previous)): + # If we're going to split the lines because of named arguments, then we + # want to split after the opening bracket as well. But not when this is + # part of a function definition. + if previous.value == '(': + # Make sure we don't split after the opening bracket if the + # continuation indent is greater than the opening bracket: + # + # a( + # b=1, + # c=2) + if (self._FitsOnLine(previous, previous.matching_bracket) and + unwrapped_line.IsSurroundedByBrackets(previous)): + # An argument to a function is a function call with named + # assigns. + return False + + column = self.column - self.stack[-1].last_space + return column > style.Get('CONTINUATION_INDENT_WIDTH') + + opening = _GetOpeningBracket(current) + if opening: + arglist_length = ( + opening.matching_bracket.total_length - opening.total_length + + self.stack[-1].indent) + return arglist_length > self.column_limit + + if (current.value not in '{)' and previous.value == '(' and + self._ArgumentListHasDictionaryEntry(current)): + return True + + if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'): + # Split before arguments in a function call or definition if the + # arguments are terminated by a comma. + opening = _GetOpeningBracket(current) + if opening and opening.previous_token and opening.previous_token.is_name: + if previous.value in '(,': + if opening.matching_bracket.previous_token.value == ',': + return True + + if ((current.is_name or current.value in {'*', '**'}) and + previous.value == ','): + # If we have a function call within an argument list and it won't fit on + # the remaining line, but it will fit on a line by itself, then go ahead + # and split before the call. + opening = _GetOpeningBracket(current) + if (opening and opening.value == '(' and opening.previous_token and + (opening.previous_token.is_name or + opening.previous_token.value in {'*', '**'})): + is_func_call = False + opening = current + while opening: + if opening.value == '(': + is_func_call = True + break + if (not (opening.is_name or opening.value in {'*', '**'}) and + opening.value != '.'): + break + opening = opening.next_token + + if is_func_call: + if (not self._FitsOnLine(current, opening.matching_bracket) or + (opening.matching_bracket.next_token and + opening.matching_bracket.next_token.value != ',' and + not opening.matching_bracket.next_token.ClosesScope())): + return True + + pprevious = previous.previous_token + if (current.is_name and pprevious and pprevious.is_name and + previous.value == '('): + if (not self._FitsOnLine(previous, previous.matching_bracket) and + _IsFunctionCallWithArguments(current)): + # There is a function call, with more than 1 argument, where the first + # argument is itself a function call with arguments. In this specific + # case, if we split after the first argument's opening '(', then the + # formatting will look bad for the rest of the arguments. E.g.: + # + # outer_function_call(inner_function_call( + # inner_arg1, inner_arg2), + # outer_arg1, outer_arg2) + # + # Instead, enforce a split before that argument to keep things looking + # good. + return True + + if (previous.OpensScope() and not current.OpensScope() and + not current.is_comment and + format_token.Subtype.SUBSCRIPT_BRACKET not in previous.subtypes): + if pprevious and not pprevious.is_keyword and not pprevious.is_name: + # We want to split if there's a comment in the container. + token = current + while token != previous.matching_bracket: + if token.is_comment: + return True + token = token.next_token + if previous.value == '(': + pptoken = previous.previous_token + if not pptoken or not pptoken.is_name: + # Split after the opening of a tuple if it doesn't fit on the current + # line and it's not a function call. + if self._FitsOnLine(previous, previous.matching_bracket): + return False + elif not self._FitsOnLine(previous, previous.matching_bracket): + if len(previous.container_elements) == 1: + return False + + elements = previous.container_elements + [previous.matching_bracket] + i = 1 + while i < len(elements): + if (not elements[i - 1].OpensScope() and + not self._FitsOnLine(elements[i - 1], elements[i])): + return True + i += 1 + + if (self.column_limit - self.column) / float(self.column_limit) < 0.3: + # Try not to squish all of the arguments off to the right. + return True + else: + # Split after the opening of a container if it doesn't fit on the + # current line. + if not self._FitsOnLine(previous, previous.matching_bracket): + return True + + ########################################################################### + # Original Formatting Splitting + # These checks rely upon the original formatting. This is in order to + # attempt to keep hand-written code in the same condition as it was before. + # However, this may cause the formatter to fail to be idempotent. + if (style.Get('SPLIT_BEFORE_BITWISE_OPERATOR') and current.value in '&|' and + previous.lineno < current.lineno): + # Retain the split before a bitwise operator. + return True + + if (current.is_comment and + previous.lineno < current.lineno - current.value.count('\n')): + # If a comment comes in the middle of an unwrapped line (like an if + # conditional with comments interspersed), then we want to split if the + # original comments were on a separate line. + return True + + return False + + def AddTokenToState(self, newline, dry_run, must_split=False): + """Add a token to the format decision state. + + Allow the heuristic to try out adding the token with and without a newline. + Later on, the algorithm will determine which one has the lowest penalty. + + Arguments: + newline: (bool) Add the token on a new line if True. + dry_run: (bool) Don't commit whitespace changes to the FormatToken if + True. + must_split: (bool) A newline was required before this token. + + Returns: + The penalty of splitting after the current token. + """ + penalty = 0 + if newline: + penalty = self._AddTokenOnNewline(dry_run, must_split) + else: + self._AddTokenOnCurrentLine(dry_run) + + penalty += self._CalculateComprehensionState(newline) + + return self.MoveStateToNextToken() + penalty + + def _AddTokenOnCurrentLine(self, dry_run): + """Puts the token on the current line. + + Appends the next token to the state and updates information necessary for + indentation. + + Arguments: + dry_run: (bool) Commit whitespace changes to the FormatToken if True. + """ + current = self.next_token + previous = current.previous_token + + spaces = current.spaces_required_before + if not dry_run: + current.AddWhitespacePrefix(newlines_before=0, spaces=spaces) + + if previous.OpensScope(): + if not current.is_comment: + # Align closing scopes that are on a newline with the opening scope: + # + # foo = [a, + # b, + # ] + self.stack[-1].closing_scope_indent = self.column - 1 + if style.Get('ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'): + self.stack[-1].closing_scope_indent += 1 + self.stack[-1].indent = self.column + spaces + else: + self.stack[-1].closing_scope_indent = ( + self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH')) + + self.column += spaces + + def _AddTokenOnNewline(self, dry_run, must_split): + """Adds a line break and necessary indentation. + + Appends the next token to the state and updates information necessary for + indentation. + + Arguments: + dry_run: (bool) Don't commit whitespace changes to the FormatToken if + True. + must_split: (bool) A newline was required before this token. + + Returns: + The split penalty for splitting after the current state. + """ + current = self.next_token + previous = current.previous_token + + self.column = self._GetNewlineColumn() + + if not dry_run: + indent_level = self.line.depth + spaces = self.column + if spaces: + spaces -= indent_level * style.Get('INDENT_WIDTH') + current.AddWhitespacePrefix( + newlines_before=1, spaces=spaces, indent_level=indent_level) + + if not current.is_comment: + self.stack[-1].last_space = self.column + self.lowest_level_on_line = self.paren_level + + if (previous.OpensScope() or + (previous.is_comment and previous.previous_token is not None and + previous.previous_token.OpensScope())): + self.stack[-1].closing_scope_indent = max( + 0, self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH')) + + self.stack[-1].split_before_closing_bracket = True + + # Calculate the split penalty. + penalty = current.split_penalty + + if must_split: + # Don't penalize for a must split. + return penalty + + if previous.is_pseudo_paren and previous.value == '(': + # Small penalty for splitting after a pseudo paren. + penalty += 50 + + # Add a penalty for each increasing newline we add, but don't penalize for + # splitting before an if-expression or list comprehension. + if current.value not in {'if', 'for'}: + last = self.stack[-1] + last.num_line_splits += 1 + penalty += ( + style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') * + last.num_line_splits) + + if current.OpensScope() and previous.OpensScope(): + # Prefer to keep opening brackets coalesced (unless it's at the beginning + # of a function call). + pprev = previous.previous_token + if not pprev or not pprev.is_name: + penalty += 10 + + return penalty + 10 + + def MoveStateToNextToken(self): + """Calculate format decision state information and move onto the next token. + + Before moving onto the next token, we first calculate the format decision + state given the current token and its formatting decisions. Then the format + decision state is set up so that the next token can be added. + + Returns: + The penalty for the number of characters over the column limit. + """ + current = self.next_token + if not current.OpensScope() and not current.ClosesScope(): + self.lowest_level_on_line = min(self.lowest_level_on_line, + self.paren_level) + + # If we encounter an opening bracket, we add a level to our stack to prepare + # for the subsequent tokens. + if current.OpensScope(): + last = self.stack[-1] + new_indent = style.Get('CONTINUATION_INDENT_WIDTH') + last.last_space + + self.stack.append(_ParenState(new_indent, self.stack[-1].last_space)) + self.paren_level += 1 + + # If we encounter a closing bracket, we can remove a level from our + # parenthesis stack. + if len(self.stack) > 1 and current.ClosesScope(): + if format_token.Subtype.DICTIONARY_KEY_PART in current.subtypes: + self.stack[-2].last_space = self.stack[-2].indent + else: + self.stack[-2].last_space = self.stack[-1].last_space + self.stack.pop() + self.paren_level -= 1 + + is_multiline_string = current.is_string and '\n' in current.value + if is_multiline_string: + # This is a multiline string. Only look at the first line. + self.column += len(current.value.split('\n')[0]) + elif not current.is_pseudo_paren: + self.column += len(current.value) + + self.next_token = self.next_token.next_token + + # Calculate the penalty for overflowing the column limit. + penalty = 0 + if (not current.is_pylint_comment and not current.is_pytype_comment and + self.column > self.column_limit): + excess_characters = self.column - self.column_limit + penalty += style.Get('SPLIT_PENALTY_EXCESS_CHARACTER') * excess_characters + + if is_multiline_string: + # If this is a multiline string, the column is actually the + # end of the last line in the string. + self.column = len(current.value.split('\n')[-1]) + + return penalty + + def _CalculateComprehensionState(self, newline): + """Makes required changes to comprehension state. + + Args: + newline: Whether the current token is to be added on a newline. + + Returns: + The penalty for the token-newline combination given the current + comprehension state. + """ + current = self.next_token + previous = current.previous_token + top_of_stack = self.comp_stack[-1] if self.comp_stack else None + penalty = 0 + + if top_of_stack is not None: + # Check if the token terminates the current comprehension. + if current == top_of_stack.closing_bracket: + last = self.comp_stack.pop() + # Lightly penalize comprehensions that are split across multiple lines. + if last.has_interior_split: + penalty += style.Get('SPLIT_PENALTY_COMPREHENSION') + + return penalty + + if newline: + top_of_stack.has_interior_split = True + + if (format_token.Subtype.COMP_EXPR in current.subtypes and + format_token.Subtype.COMP_EXPR not in previous.subtypes): + self.comp_stack.append(object_state.ComprehensionState(current)) + return penalty + + if (current.value == 'for' and + format_token.Subtype.COMP_FOR in current.subtypes): + if top_of_stack.for_token is not None: + # Treat nested comprehensions like normal comp_if expressions. + # Example: + # my_comp = [ + # a.qux + b.qux + # for a in foo + # --> for b in bar <-- + # if a.zut + b.zut + # ] + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and + top_of_stack.has_split_at_for != newline and + (top_of_stack.has_split_at_for or + not top_of_stack.HasTrivialExpr())): + penalty += split_penalty.UNBREAKABLE + else: + top_of_stack.for_token = current + top_of_stack.has_split_at_for = newline + + # Try to keep trivial expressions on the same line as the comp_for. + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and newline and + top_of_stack.HasTrivialExpr()): + penalty += split_penalty.CONNECTED + + if (format_token.Subtype.COMP_IF in current.subtypes and + format_token.Subtype.COMP_IF not in previous.subtypes): + # Penalize breaking at comp_if when it doesn't match the newline structure + # in the rest of the comprehension. + if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and + top_of_stack.has_split_at_for != newline and + (top_of_stack.has_split_at_for or not top_of_stack.HasTrivialExpr())): + penalty += split_penalty.UNBREAKABLE + + return penalty + + def _GetNewlineColumn(self): + """Return the new column on the newline.""" + current = self.next_token + previous = current.previous_token + top_of_stack = self.stack[-1] + + if current.spaces_required_before > 2 or self.line.disable: + return current.spaces_required_before + + if current.OpensScope(): + return top_of_stack.indent if self.paren_level else self.first_indent + + if current.ClosesScope(): + if (previous.OpensScope() or + (previous.is_comment and previous.previous_token is not None and + previous.previous_token.OpensScope())): + return max(0, + top_of_stack.indent - style.Get('CONTINUATION_INDENT_WIDTH')) + return top_of_stack.closing_scope_indent + + if (previous and previous.is_string and current.is_string and + format_token.Subtype.DICTIONARY_VALUE in current.subtypes): + return previous.column + + if style.Get('INDENT_DICTIONARY_VALUE'): + if previous and (previous.value == ':' or previous.is_pseudo_paren): + if format_token.Subtype.DICTIONARY_VALUE in current.subtypes: + return top_of_stack.indent + + if (_IsCompoundStatement(self.line.first) and + (not style.Get('DEDENT_CLOSING_BRACKETS') or + style.Get('SPLIT_BEFORE_FIRST_ARGUMENT'))): + token_indent = ( + len(self.line.first.whitespace_prefix.split('\n')[-1]) + + style.Get('INDENT_WIDTH')) + if token_indent == top_of_stack.indent: + return top_of_stack.indent + style.Get('CONTINUATION_INDENT_WIDTH') + + return top_of_stack.indent + + def _FitsOnLine(self, start, end): + """Determines if line between start and end can fit on the current line.""" + length = end.total_length - start.total_length + if not start.is_pseudo_paren: + length += len(start.value) + return length + self.column <= self.column_limit + + def _EachDictEntryFitsOnOneLine(self, opening): + """Determine if each dict elems can fit on one line.""" + + def PreviousNonCommentToken(tok): + tok = tok.previous_token + while tok.is_comment: + tok = tok.previous_token + return tok + + def ImplicitStringConcatenation(tok): + num_strings = 0 + if tok.is_pseudo_paren: + tok = tok.next_token + while tok.is_string: + num_strings += 1 + tok = tok.next_token + return num_strings > 1 + + closing = opening.matching_bracket + entry_start = opening.next_token + current = opening.next_token.next_token + + while current and current != closing: + if format_token.Subtype.DICTIONARY_KEY in current.subtypes: + prev = PreviousNonCommentToken(current) + length = prev.total_length - entry_start.total_length + length += len(entry_start.value) + if length + self.stack[-2].indent >= self.column_limit: + return False + entry_start = current + if current.OpensScope(): + if ((current.value == '{' or + (current.is_pseudo_paren and current.next_token.value == '{') and + format_token.Subtype.DICTIONARY_VALUE in current.subtypes) or + ImplicitStringConcatenation(current)): + # A dictionary entry that cannot fit on a single line shouldn't matter + # to this calculation. If it can't fit on a single line, then the + # opening should be on the same line as the key and the rest on + # newlines after it. But the other entries should be on single lines + # if possible. + if current.matching_bracket: + current = current.matching_bracket + while current: + if current == closing: + return True + if format_token.Subtype.DICTIONARY_KEY in current.subtypes: + entry_start = current + break + current = current.next_token + else: + current = current.matching_bracket + else: + current = current.next_token + + # At this point, current is the closing bracket. Go back one to get the the + # end of the dictionary entry. + current = PreviousNonCommentToken(current) + length = current.total_length - entry_start.total_length + length += len(entry_start.value) + return length + self.stack[-2].indent <= self.column_limit + + def _ArgumentListHasDictionaryEntry(self, token): + """Check if the function argument list has a dictionary as an arg.""" + if _IsArgumentToFunction(token): + while token: + if token.value == '{': + length = token.matching_bracket.total_length - token.total_length + return length + self.stack[-2].indent > self.column_limit + if token.ClosesScope(): + break + if token.OpensScope(): + token = token.matching_bracket + token = token.next_token + return False + + +_COMPOUND_STMTS = frozenset( + {'for', 'while', 'if', 'elif', 'with', 'except', 'def', 'class'}) + + +def _IsCompoundStatement(token): + if token.value == 'async': + token = token.next_token + return token.value in _COMPOUND_STMTS + + +def _IsFunctionDef(token): + if token.value == 'async': + token = token.next_token + return token.value == 'def' + + +def _IsFunctionCallWithArguments(token): + while token: + if token.value == '(': + token = token.next_token + return token and token.value != ')' + elif token.name not in {'NAME', 'DOT', 'EQUAL'}: + break + token = token.next_token + return False + + +def _IsArgumentToFunction(token): + bracket = unwrapped_line.IsSurroundedByBrackets(token) + if not bracket or bracket.value != '(': + return False + previous = bracket.previous_token + return previous and previous.is_name + + +def _GetLengthOfSubtype(token, subtype, exclude=None): + current = token + while (current.next_token and subtype in current.subtypes and + (exclude is None or exclude not in current.subtypes)): + current = current.next_token + return current.total_length - token.total_length + 1 + + +def _GetOpeningBracket(current): + """Get the opening bracket containing the current token.""" + if current.matching_bracket and not current.is_pseudo_paren: + return current.matching_bracket + while current: + if current.ClosesScope(): + current = current.matching_bracket + elif current.is_pseudo_paren: + current = current.previous_token + elif current.OpensScope(): + return current + current = current.previous_token + return None + + +def _LastTokenInLine(current): + while not current.is_comment and current.next_token: + current = current.next_token + return current + + +def _IsFunctionDefinition(current): + prev = current.previous_token + return (current.value == '(' and prev and + format_token.Subtype.FUNC_DEF in prev.subtypes) + + +def _IsLastScopeInLine(current): + while current: + current = current.next_token + if current and current.OpensScope(): + return False + return True + + +def _IsSingleElementTuple(token): + """Check if it's a single-element tuple.""" + close = token.matching_bracket + token = token.next_token + num_commas = 0 + while token != close: + if token.value == ',': + num_commas += 1 + if token.OpensScope(): + token = token.matching_bracket + else: + token = token.next_token + return num_commas == 1 + + +def _ScopeHasNoCommas(token): + """Check if the scope has no commas.""" + close = token.matching_bracket + token = token.next_token + while token != close: + if token.value == ',': + return False + if token.OpensScope(): + token = token.matching_bracket + else: + token = token.next_token + return True + + +class _ParenState(object): + """Maintains the state of the bracket enclosures. + + A stack of _ParenState objects are kept so that we know how to indent relative + to the brackets. + + Attributes: + indent: The column position to which a specified parenthesis level needs to + be indented. + last_space: The column position of the last space on each level. + split_before_closing_bracket: Whether a newline needs to be inserted before + the closing bracket. We only want to insert a newline before the closing + bracket if there also was a newline after the beginning left bracket. + num_line_splits: Number of line splits this _ParenState contains already. + Each subsequent line split gets an increasing penalty. + """ + + # TODO(morbo): This doesn't track "bin packing." + + def __init__(self, indent, last_space): + self.indent = indent + self.last_space = last_space + self.closing_scope_indent = 0 + self.split_before_closing_bracket = False + self.num_line_splits = 0 + + def Clone(self): + state = _ParenState(self.indent, self.last_space) + state.closing_scope_indent = self.closing_scope_indent + state.split_before_closing_bracket = self.split_before_closing_bracket + state.num_line_splits = self.num_line_splits + return state + + def __repr__(self): + return '[indent::%d, last_space::%d, closing_scope_indent::%d]' % ( + self.indent, self.last_space, self.closing_scope_indent) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash((self.indent, self.last_space, self.closing_scope_indent, + self.split_before_closing_bracket, self.num_line_splits)) diff --git a/yapf/yapflib/format_token.py b/yapf/yapflib/format_token.py new file mode 100644 index 0000000..79dced4 --- /dev/null +++ b/yapf/yapflib/format_token.py @@ -0,0 +1,339 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pytree nodes with extra formatting information. + +This is a thin wrapper around a pytree.Leaf node. +""" + +import keyword +import re + +from lib2to3.pgen2 import token + +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils +from yapf.yapflib import style + +CONTINUATION = token.N_TOKENS + + +class Subtype(object): + """Subtype information about tokens. + + Gleaned from parsing the code. Helps determine the best formatting. + """ + NONE = 0 + UNARY_OPERATOR = 1 + BINARY_OPERATOR = 2 + SUBSCRIPT_COLON = 3 + SUBSCRIPT_BRACKET = 4 + DEFAULT_OR_NAMED_ASSIGN = 5 + DEFAULT_OR_NAMED_ASSIGN_ARG_LIST = 6 + VARARGS_LIST = 7 + VARARGS_STAR = 8 + KWARGS_STAR_STAR = 9 + ASSIGN_OPERATOR = 10 + DICTIONARY_KEY = 11 + DICTIONARY_KEY_PART = 12 + DICTIONARY_VALUE = 13 + DICT_SET_GENERATOR = 14 + COMP_EXPR = 21 + COMP_FOR = 15 + COMP_IF = 16 + FUNC_DEF = 17 + DECORATOR = 18 + TYPED_NAME = 19 + TYPED_NAME_ARG_LIST = 20 + + +def _TabbedContinuationAlignPadding(spaces, align_style, tab_width, + continuation_indent_width): + """Build padding string for continuation alignment in tabbed indentation. + + Arguments: + spaces: (int) The number of spaces to place before the token for alignment. + align_style: (str) The alignment style for continuation lines. + tab_width: (int) Number of columns of each tab character. + continuation_indent_width: (int) Indent columns for line continuations. + + Returns: + A padding string for alignment with style specified by align_style option. + """ + if align_style == 'FIXED': + if spaces > 0: + return '\t' * int(continuation_indent_width / tab_width) + return '' + elif align_style == 'VALIGN-RIGHT': + return '\t' * int((spaces + tab_width - 1) / tab_width) + return ' ' * spaces + + +class FormatToken(object): + """A wrapper around pytree Leaf nodes. + + This represents the token plus additional information useful for reformatting + the code. + + Attributes: + next_token: The token in the unwrapped line after this token or None if this + is the last token in the unwrapped line. + previous_token: The token in the unwrapped line before this token or None if + this is the first token in the unwrapped line. + matching_bracket: If a bracket token ('[', '{', or '(') the matching + bracket. + container_opening: If the object is in a container, this points to its + opening bracket. + container_elements: If this is the start of a container, a list of the + elements in the container. + whitespace_prefix: The prefix for the whitespace. + spaces_required_before: The number of spaces required before a token. This + is a lower-bound for the formatter and not a hard requirement. For + instance, a comment may have n required spaces before it. But the + formatter won't place n spaces before all comments. Only those that are + moved to the end of a line of code. The formatter may use different + spacing when appropriate. + can_break_before: True if we're allowed to break before this token. + must_break_before: True if we're required to break before this token. + total_length: The total length of the unwrapped line up to and including + whitespace and this token. However, this doesn't include the initial + indentation amount. + split_penalty: The penalty for splitting the line before this token. + """ + + def __init__(self, node): + """Constructor. + + Arguments: + node: (pytree.Leaf) The node that's being wrapped. + """ + self.node = node + self.next_token = None + self.previous_token = None + self.matching_bracket = None + self.container_opening = None + self.container_elements = [] + self.whitespace_prefix = '' + self.can_break_before = False + self.must_break_before = False + self.total_length = 0 # TODO(morbo): Think up a better name. + self.split_penalty = 0 + + if self.is_comment: + self.spaces_required_before = style.Get('SPACES_BEFORE_COMMENT') + else: + self.spaces_required_before = 0 + + if self.is_continuation: + self.value = self.node.value.rstrip() + else: + self.value = self.node.value + + def AddWhitespacePrefix(self, newlines_before, spaces=0, indent_level=0): + """Register a token's whitespace prefix. + + This is the whitespace that will be output before a token's string. + + Arguments: + newlines_before: (int) The number of newlines to place before the token. + spaces: (int) The number of spaces to place before the token. + indent_level: (int) The indentation level. + """ + if style.Get('USE_TABS'): + if newlines_before > 0: + indent_before = '\t' * indent_level + _TabbedContinuationAlignPadding( + spaces, style.Get('CONTINUATION_ALIGN_STYLE'), + style.Get('INDENT_WIDTH'), style.Get('CONTINUATION_INDENT_WIDTH')) + else: + indent_before = '\t' * indent_level + ' ' * spaces + else: + indent_before = ( + ' ' * indent_level * style.Get('INDENT_WIDTH') + ' ' * spaces) + + if self.is_comment: + comment_lines = [s.lstrip() for s in self.value.splitlines()] + self.node.value = ('\n' + indent_before).join(comment_lines) + + # Update our own value since we are changing node value + self.value = self.node.value + + if not self.whitespace_prefix: + self.whitespace_prefix = ( + '\n' * (self.newlines or newlines_before) + indent_before) + else: + self.whitespace_prefix += indent_before + + def AdjustNewlinesBefore(self, newlines_before): + """Change the number of newlines before this token.""" + self.whitespace_prefix = ( + '\n' * newlines_before + self.whitespace_prefix.lstrip('\n')) + + def RetainHorizontalSpacing(self, first_column, depth): + """Retains a token's horizontal spacing.""" + previous = self.previous_token + if not previous: + return + + if previous.is_pseudo_paren: + previous = previous.previous_token + if not previous: + return + + cur_lineno = self.lineno + prev_lineno = previous.lineno + if previous.is_multiline_string: + prev_lineno += previous.value.count('\n') + + if (cur_lineno != prev_lineno or + (previous.is_pseudo_paren and previous.value != ')' and + cur_lineno != previous.previous_token.lineno)): + self.spaces_required_before = ( + self.column - first_column + depth * style.Get('INDENT_WIDTH')) + return + + cur_column = self.node.column + prev_column = previous.node.column + prev_len = len(previous.value) + + if previous.is_pseudo_paren and previous.value == ')': + prev_column -= 1 + prev_len = 0 + + if previous.is_multiline_string: + prev_len = len(previous.value.split('\n')[-1]) + if '\n' in previous.value: + prev_column = 0 # Last line starts in column 0. + + self.spaces_required_before = cur_column - (prev_column + prev_len) + + def OpensScope(self): + return self.value in pytree_utils.OPENING_BRACKETS + + def ClosesScope(self): + return self.value in pytree_utils.CLOSING_BRACKETS + + def __repr__(self): + msg = 'FormatToken(name={0}, value={1}'.format(self.name, self.value) + msg += ', pseudo)' if self.is_pseudo_paren else ')' + return msg + + @property + @py3compat.lru_cache() + def node_split_penalty(self): + """Split penalty attached to the pytree node of this token.""" + return pytree_utils.GetNodeAnnotation( + self.node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) + + @property + def newlines(self): + """The number of newlines needed before this token.""" + return pytree_utils.GetNodeAnnotation(self.node, + pytree_utils.Annotation.NEWLINES) + + @property + def must_split(self): + """Return true if the token requires a split before it.""" + return pytree_utils.GetNodeAnnotation(self.node, + pytree_utils.Annotation.MUST_SPLIT) + + @property + def column(self): + """The original column number of the node in the source.""" + return self.node.column + + @property + def lineno(self): + """The original line number of the node in the source.""" + return self.node.lineno + + @property + @py3compat.lru_cache() + def subtypes(self): + """Extra type information for directing formatting.""" + value = pytree_utils.GetNodeAnnotation(self.node, + pytree_utils.Annotation.SUBTYPE) + return [Subtype.NONE] if value is None else value + + @property + @py3compat.lru_cache() + def is_binary_op(self): + """Token is a binary operator.""" + return Subtype.BINARY_OPERATOR in self.subtypes + + @property + @py3compat.lru_cache() + def name(self): + """A string representation of the node's name.""" + return pytree_utils.NodeName(self.node) + + @property + def is_comment(self): + return self.node.type == token.COMMENT + + @property + def is_continuation(self): + return self.node.type == CONTINUATION + + @property + @py3compat.lru_cache() + def is_keyword(self): + return keyword.iskeyword(self.value) + + @property + @py3compat.lru_cache() + def is_name(self): + return self.node.type == token.NAME and not self.is_keyword + + @property + def is_number(self): + return self.node.type == token.NUMBER + + @property + def is_string(self): + return self.node.type == token.STRING + + @property + @py3compat.lru_cache() + def is_multiline_string(self): + """A multiline string.""" + if py3compat.PY3: + prefix = '(' + prefix += 'r|u|R|U|f|F|fr|Fr|fR|FR|rf|rF|Rf|RF' # strings + prefix += '|b|B|br|Br|bR|BR|rb|rB|Rb|RB' # bytes + prefix += ')?' + else: + prefix = '[uUbB]?[rR]?' + + regex = r'^{prefix}(?P<delim>"""|\'\'\').*(?P=delim)$'.format(prefix=prefix) + return (self.is_string and + re.match(regex, self.value, re.DOTALL) is not None) + + @property + @py3compat.lru_cache() + def is_docstring(self): + return self.is_multiline_string and not self.node.prev_sibling + + @property + @py3compat.lru_cache() + def is_pseudo_paren(self): + return hasattr(self.node, 'is_pseudo') and self.node.is_pseudo + + @property + def is_pylint_comment(self): + return self.is_comment and re.match(r'#.*\bpylint:\s*(disable|enable)=', + self.value) + + @property + def is_pytype_comment(self): + return self.is_comment and re.match(r'#.*\bpytype:\s*(disable|enable)=', + self.value) diff --git a/yapf/yapflib/identify_container.py b/yapf/yapflib/identify_container.py new file mode 100644 index 0000000..5c5fc5b --- /dev/null +++ b/yapf/yapflib/identify_container.py @@ -0,0 +1,67 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Identify containers for lib2to3 trees. + +This module identifies containers and the elements in them. Each element points +to the opening bracket and vice-versa. + + IdentifyContainers(): the main function exported by this module. +""" + +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor + + +def IdentifyContainers(tree): + """Run the identify containers visitor over the tree, modifying it in place. + + Arguments: + tree: the top-level pytree node to annotate with subtypes. + """ + identify_containers = _IdentifyContainers() + identify_containers.Visit(tree) + + +class _IdentifyContainers(pytree_visitor.PyTreeVisitor): + """_IdentifyContainers - see file-level docstring for detailed description.""" + + def Visit_trailer(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + + if len(node.children) != 3: + return + if pytree_utils.NodeName(node.children[0]) != 'LPAR': + return + + if pytree_utils.NodeName(node.children[1]) == 'arglist': + for child in node.children[1].children: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(child), node.children[0]) + else: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(node.children[1]), node.children[0]) + + def Visit_atom(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + + if len(node.children) != 3: + return + if pytree_utils.NodeName(node.children[0]) != 'LPAR': + return + + for child in node.children[1].children: + pytree_utils.SetOpeningBracket( + pytree_utils.FirstLeafNode(child), node.children[0]) diff --git a/yapf/yapflib/line_joiner.py b/yapf/yapflib/line_joiner.py new file mode 100644 index 0000000..84346c2 --- /dev/null +++ b/yapf/yapflib/line_joiner.py @@ -0,0 +1,109 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Join unwrapped lines together. + +Determine how many lines can be joined into one line. For instance, we could +join these statements into one line: + + if a == 42: + continue + +like this: + + if a == 42: continue + +There are a few restrictions: + + 1. The lines should have been joined in the original source. + 2. The joined lines must not go over the column boundary if placed on the same + line. + 3. They need to be very simple statements. + +Note: Because we don't allow the use of a semicolon to separate statements, it +follows that there can only be at most two lines to join. +""" + +from yapf.yapflib import style + +_CLASS_OR_FUNC = frozenset({'def', 'class'}) + + +def CanMergeMultipleLines(lines, last_was_merged=False): + """Determine if multiple lines can be joined into one. + + Arguments: + lines: (list of UnwrappedLine) This is a splice of UnwrappedLines from the + full code base. + last_was_merged: (bool) The last line was merged. + + Returns: + True if two consecutive lines can be joined together. In reality, this will + only happen if two consecutive lines can be joined, due to the style guide. + """ + # The indentation amount for the starting line (number of spaces). + indent_amt = lines[0].depth * style.Get('INDENT_WIDTH') + if len(lines) == 1 or indent_amt > style.Get('COLUMN_LIMIT'): + return False + + if (len(lines) >= 3 and lines[2].depth >= lines[1].depth and + lines[0].depth != lines[2].depth): + # If lines[2]'s depth is greater than or equal to line[1]'s depth, we're not + # looking at a single statement (e.g., if-then, while, etc.). A following + # line with the same depth as the first line isn't part of the lines we + # would want to combine. + return False # Don't merge more than two lines together. + + if lines[0].first.value in _CLASS_OR_FUNC: + # Don't join lines onto the starting line of a class or function. + return False + + limit = style.Get('COLUMN_LIMIT') - indent_amt + if lines[0].last.total_length < limit: + limit -= lines[0].last.total_length + + if lines[0].first.value == 'if': + return _CanMergeLineIntoIfStatement(lines, limit) + if last_was_merged and lines[0].first.value in {'elif', 'else'}: + return _CanMergeLineIntoIfStatement(lines, limit) + + # TODO(morbo): Other control statements? + + return False + + +def _CanMergeLineIntoIfStatement(lines, limit): + """Determine if we can merge a short if-then statement into one line. + + Two lines of an if-then statement can be merged if they were that way in the + original source, fit on the line without going over the column limit, and are + considered "simple" statements --- typically statements like 'pass', + 'continue', and 'break'. + + Arguments: + lines: (list of UnwrappedLine) The lines we are wanting to merge. + limit: (int) The amount of space remaining on the line. + + Returns: + True if the lines can be merged, False otherwise. + """ + if len(lines[1].tokens) == 1 and lines[1].last.is_multiline_string: + # This might be part of a multiline shebang. + return True + if lines[0].lineno != lines[1].lineno: + # Don't merge lines if the original lines weren't merged. + return False + if lines[1].last.total_length >= limit: + # Don't merge lines if the result goes over the column limit. + return False + return style.Get('JOIN_MULTIPLE_LINES') diff --git a/yapf/yapflib/object_state.py b/yapf/yapflib/object_state.py new file mode 100644 index 0000000..dded7c4 --- /dev/null +++ b/yapf/yapflib/object_state.py @@ -0,0 +1,80 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Represents the state of Python objects being formatted. + +Objects (e.g., list comprehensions, dictionaries, etc.) have specific +requirements on how they're formatted. These state objects keep track of these +requirements. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class ComprehensionState(object): + """Maintains the state of list comprehension formatting decisions. + + A stack of ComprehensionState objects are kept to ensure that list + comprehensions are wrapped with well-defined rules. + + Attributes: + expr_token: The first token in the comprehension. + for_token: The first 'for' token of the comprehension. + has_split_at_for: Whether there is a newline immediately before the + for_token. + has_interior_split: Whether there is a newline within the comprehension. + That is, a split somewhere after expr_token or before closing_bracket. + """ + + def __init__(self, expr_token): + self.expr_token = expr_token + self.for_token = None + self.has_split_at_for = False + self.has_interior_split = False + + def HasTrivialExpr(self): + """Returns whether the comp_expr is "trivial" i.e. is a single token.""" + return self.expr_token.next_token.value == 'for' + + @property + def opening_bracket(self): + return self.expr_token.previous_token + + @property + def closing_bracket(self): + return self.opening_bracket.matching_bracket + + def Clone(self): + clone = ComprehensionState(self.expr_token) + clone.for_token = self.for_token + clone.has_split_at_for = self.has_split_at_for + clone.has_interior_split = self.has_interior_split + return clone + + def __repr__(self): + return ('[opening_bracket::%s, for_token::%s, has_split_at_for::%s,' + ' has_interior_split::%s, has_trivial_expr::%s]' % + (self.opening_bracket, self.for_token, self.has_split_at_for, + self.has_interior_split, self.HasTrivialExpr())) + + def __eq__(self, other): + return hash(self) == hash(other) + + def __ne__(self, other): + return not self == other + + def __hash__(self, *args, **kwargs): + return hash((self.expr_token, self.for_token, self.has_split_at_for, + self.has_interior_split)) diff --git a/yapf/yapflib/py3compat.py b/yapf/yapflib/py3compat.py new file mode 100644 index 0000000..c66d6c6 --- /dev/null +++ b/yapf/yapflib/py3compat.py @@ -0,0 +1,118 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for Python2 / Python3 compatibility.""" + +import io +import os +import sys + +PY3 = sys.version_info[0] >= 3 +PY36 = sys.version_info[0] >= 3 and sys.version_info[1] >= 6 + +if PY3: + StringIO = io.StringIO + BytesIO = io.BytesIO + + import codecs + + def open_with_encoding(filename, mode, encoding, newline=''): # pylint: disable=unused-argument + return codecs.open(filename, mode=mode, encoding=encoding) + + import functools + lru_cache = functools.lru_cache + + range = range + ifilter = filter + + def raw_input(): + wrapper = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') + return wrapper.buffer.raw.readall().decode('utf-8') + + import configparser + + # Mappings from strings to booleans (such as '1' to True, 'false' to False, + # etc.) + CONFIGPARSER_BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES +else: + import __builtin__ + import cStringIO + StringIO = BytesIO = cStringIO.StringIO + + open_with_encoding = io.open + + # Python 2.7 doesn't have a native LRU cache, so do nothing. + def lru_cache(maxsize=128, typed=False): + + def fake_wrapper(user_function): + return user_function + + return fake_wrapper + + range = xrange + + from itertools import ifilter + raw_input = raw_input + + import ConfigParser as configparser + CONFIGPARSER_BOOLEAN_STATES = configparser.ConfigParser._boolean_states # pylint: disable=protected-access + + +def EncodeAndWriteToStdout(s, encoding='utf-8'): + """Encode the given string and emit to stdout. + + The string may contain non-ascii characters. This is a problem when stdout is + redirected, because then Python doesn't know the encoding and we may get a + UnicodeEncodeError. + + Arguments: + s: (string) The string to encode. + encoding: (string) The encoding of the string. + """ + if PY3: + sys.stdout.buffer.write(s.encode(encoding)) + elif sys.platform == 'win32': + # On python 2 and Windows universal newline transformation will be in + # effect on stdout. Python 2 will not let us avoid the easily because + # it happens based on whether the file handle is opened in O_BINARY or + # O_TEXT state. However we can tell Windows itself to change the current + # mode, and python 2 will follow suit. However we must take care to change + # the mode on the actual external stdout not just the current sys.stdout + # which may have been monkey-patched inside the python environment. + import msvcrt # pylint: disable=g-import-not-at-top + if sys.__stdout__ is sys.stdout: + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + sys.stdout.write(s.encode(encoding)) + else: + sys.stdout.write(s.encode(encoding)) + + +if PY3: + basestring = str + unicode = str # pylint: disable=redefined-builtin,invalid-name +else: + basestring = basestring + + def unicode(s): # pylint: disable=invalid-name + """Force conversion of s to unicode.""" + return __builtin__.unicode(s, 'utf-8') + + +# In Python 3.2+, readfp is deprecated in favor of read_file, which doesn't +# exist in Python 2 yet. To avoid deprecation warnings, subclass ConfigParser to +# fix this - now read_file works across all Python versions we care about. +class ConfigParser(configparser.ConfigParser): + if not PY3: + + def read_file(self, fp, source=None): + self.readfp(fp, filename=source) diff --git a/yapf/yapflib/pytree_unwrapper.py b/yapf/yapflib/pytree_unwrapper.py new file mode 100644 index 0000000..0d371ae --- /dev/null +++ b/yapf/yapflib/pytree_unwrapper.py @@ -0,0 +1,386 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTreeUnwrapper - produces a list of unwrapped lines from a pytree. + +[for a description of what an unwrapped line is, see unwrapped_line.py] + +This is a pytree visitor that goes over a parse tree and produces a list of +UnwrappedLine containers from it, each with its own depth and containing all +the tokens that could fit on the line if there were no maximal line-length +limitations. + +Note: a precondition to running this visitor and obtaining correct results is +for the tree to have its comments spliced in as nodes. Prefixes are ignored. + +For most uses, the convenience function UnwrapPyTree should be sufficient. +""" + +# The word "token" is overloaded within this module, so for clarity rename +# the imported pgen2.token module. +from lib2to3 import pytree +from lib2to3.pgen2 import token as grammar_token + +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import split_penalty +from yapf.yapflib import style +from yapf.yapflib import unwrapped_line + + +def UnwrapPyTree(tree): + """Create and return a list of unwrapped lines from the given pytree. + + Arguments: + tree: the top-level pytree node to unwrap. + + Returns: + A list of UnwrappedLine objects. + """ + unwrapper = PyTreeUnwrapper() + unwrapper.Visit(tree) + uwlines = unwrapper.GetUnwrappedLines() + uwlines.sort(key=lambda x: x.lineno) + return uwlines + + +# Grammar tokens considered as whitespace for the purpose of unwrapping. +_WHITESPACE_TOKENS = frozenset([ + grammar_token.NEWLINE, grammar_token.DEDENT, grammar_token.INDENT, + grammar_token.ENDMARKER +]) + + +class PyTreeUnwrapper(pytree_visitor.PyTreeVisitor): + """PyTreeUnwrapper - see file-level docstring for detailed description. + + Note: since this implements PyTreeVisitor and node names in lib2to3 are + underscore_separated, the visiting methods of this class are named as + Visit_node_name. invalid-name pragmas are added to each such method to silence + a style warning. This is forced on us by the usage of lib2to3, and re-munging + method names to make them different from actual node names sounded like a + confusing and brittle affair that wasn't worth it for this small & controlled + deviation from the style guide. + + To understand the connection between visitor methods in this class, some + familiarity with the Python grammar is required. + """ + + def __init__(self): + # A list of all unwrapped lines finished visiting so far. + self._unwrapped_lines = [] + + # Builds up a "current" unwrapped line while visiting pytree nodes. Some + # nodes will finish a line and start a new one. + self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(0) + + # Current indentation depth. + self._cur_depth = 0 + + def GetUnwrappedLines(self): + """Fetch the result of the tree walk. + + Note: only call this after visiting the whole tree. + + Returns: + A list of UnwrappedLine objects. + """ + # Make sure the last line that was being populated is flushed. + self._StartNewLine() + return self._unwrapped_lines + + def _StartNewLine(self): + """Finish current line and start a new one. + + Place the currently accumulated line into the _unwrapped_lines list and + start a new one. + """ + if self._cur_unwrapped_line.tokens: + self._unwrapped_lines.append(self._cur_unwrapped_line) + _MatchBrackets(self._cur_unwrapped_line) + _AdjustSplitPenalty(self._cur_unwrapped_line) + self._cur_unwrapped_line = unwrapped_line.UnwrappedLine(self._cur_depth) + + _STMT_TYPES = frozenset({ + 'if_stmt', + 'while_stmt', + 'for_stmt', + 'try_stmt', + 'expect_clause', + 'with_stmt', + 'funcdef', + 'classdef', + }) + + # pylint: disable=invalid-name,missing-docstring + def Visit_simple_stmt(self, node): + # A 'simple_stmt' conveniently represents a non-compound Python statement, + # i.e. a statement that does not contain other statements. + + # When compound nodes have a single statement as their suite, the parser + # can leave it in the tree directly without creating a suite. But we have + # to increase depth in these cases as well. However, don't increase the + # depth of we have a simple_stmt that's a comment node. This represents a + # standalone comment and in the case of it coming directly after the + # funcdef, it is a "top" comment for the whole function. + # TODO(eliben): add more relevant compound statements here. + single_stmt_suite = ( + node.parent and pytree_utils.NodeName(node.parent) in self._STMT_TYPES) + is_comment_stmt = pytree_utils.IsCommentStatement(node) + if single_stmt_suite and not is_comment_stmt: + self._cur_depth += 1 + self._StartNewLine() + self.DefaultNodeVisit(node) + if single_stmt_suite and not is_comment_stmt: + self._cur_depth -= 1 + + def _VisitCompoundStatement(self, node, substatement_names): + """Helper for visiting compound statements. + + Python compound statements serve as containers for other statements. Thus, + when we encounter a new compound statement we start a new unwrapped line. + + Arguments: + node: the node to visit. + substatement_names: set of node names. A compound statement will be + recognized as a NAME node with a name in this set. + """ + for child in node.children: + # A pytree is structured in such a way that a single 'if_stmt' node will + # contain all the 'if', 'elif' and 'else' nodes as children (similar + # structure applies to 'while' statements, 'try' blocks, etc). Therefore, + # we visit all children here and create a new line before the requested + # set of nodes. + if (child.type == grammar_token.NAME and + child.value in substatement_names): + self._StartNewLine() + self.Visit(child) + + _IF_STMT_ELEMS = frozenset({'if', 'else', 'elif'}) + + def Visit_if_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._IF_STMT_ELEMS) + + _WHILE_STMT_ELEMS = frozenset({'while', 'else'}) + + def Visit_while_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._WHILE_STMT_ELEMS) + + _FOR_STMT_ELEMS = frozenset({'for', 'else'}) + + def Visit_for_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._FOR_STMT_ELEMS) + + _TRY_STMT_ELEMS = frozenset({'try', 'except', 'else', 'finally'}) + + def Visit_try_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._TRY_STMT_ELEMS) + + _EXCEPT_STMT_ELEMS = frozenset({'except'}) + + def Visit_except_clause(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._EXCEPT_STMT_ELEMS) + + _FUNC_DEF_ELEMS = frozenset({'def'}) + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._FUNC_DEF_ELEMS) + + def Visit_async_funcdef(self, node): # pylint: disable=invalid-name + self._StartNewLine() + index = 0 + for child in node.children: + index += 1 + self.Visit(child) + if pytree_utils.NodeName(child) == 'ASYNC': + break + for child in node.children[index].children: + self.Visit(child) + + _CLASS_DEF_ELEMS = frozenset({'class'}) + + def Visit_classdef(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._CLASS_DEF_ELEMS) + + def Visit_async_stmt(self, node): # pylint: disable=invalid-name + self._StartNewLine() + index = 0 + for child in node.children: + index += 1 + self.Visit(child) + if pytree_utils.NodeName(child) == 'ASYNC': + break + for child in node.children[index].children: + self.Visit(child) + + def Visit_decorator(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + if (pytree_utils.NodeName(child) == 'COMMENT' and + child == node.children[0]): + self._StartNewLine() + + def Visit_decorators(self, node): # pylint: disable=invalid-name + for child in node.children: + self._StartNewLine() + self.Visit(child) + + def Visit_decorated(self, node): # pylint: disable=invalid-name + for child in node.children: + self._StartNewLine() + self.Visit(child) + + _WITH_STMT_ELEMS = frozenset({'with'}) + + def Visit_with_stmt(self, node): # pylint: disable=invalid-name + self._VisitCompoundStatement(node, self._WITH_STMT_ELEMS) + + def Visit_suite(self, node): # pylint: disable=invalid-name + # A 'suite' starts a new indentation level in Python. + self._cur_depth += 1 + self._StartNewLine() + self.DefaultNodeVisit(node) + self._cur_depth -= 1 + + def Visit_listmaker(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def Visit_import_as_names(self, node): # pylint: disable=invalid-name + if node.prev_sibling.value == '(': + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def Visit_testlist_gexp(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def Visit_arglist(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def Visit_typedargslist(self, node): # pylint: disable=invalid-name + _DetermineMustSplitAnnotation(node) + self.DefaultNodeVisit(node) + + def DefaultLeafVisit(self, leaf): + """Default visitor for tree leaves. + + A tree leaf is always just gets appended to the current unwrapped line. + + Arguments: + leaf: the leaf to visit. + """ + if leaf.type in _WHITESPACE_TOKENS: + self._StartNewLine() + elif leaf.type != grammar_token.COMMENT or leaf.value.strip(): + # Add non-whitespace tokens and comments that aren't empty. + self._cur_unwrapped_line.AppendNode(leaf) + + +_BRACKET_MATCH = {')': '(', '}': '{', ']': '['} + + +def _MatchBrackets(uwline): + """Visit the node and match the brackets. + + For every open bracket ('[', '{', or '('), find the associated closing bracket + and "match" them up. I.e., save in the token a pointer to its associated open + or close bracket. + + Arguments: + uwline: (UnwrappedLine) An unwrapped line. + """ + bracket_stack = [] + for token in uwline.tokens: + if token.value in pytree_utils.OPENING_BRACKETS: + bracket_stack.append(token) + elif token.value in pytree_utils.CLOSING_BRACKETS: + bracket_stack[-1].matching_bracket = token + token.matching_bracket = bracket_stack[-1] + bracket_stack.pop() + + for bracket in bracket_stack: + if id(pytree_utils.GetOpeningBracket(token.node)) == id(bracket.node): + bracket.container_elements.append(token) + token.container_opening = bracket + + +def _AdjustSplitPenalty(uwline): + """Visit the node and adjust the split penalties if needed. + + A token shouldn't be split if it's not within a bracket pair. Mark any token + that's not within a bracket pair as "unbreakable". + + Arguments: + uwline: (UnwrappedLine) An unwrapped line. + """ + bracket_level = 0 + for index, token in enumerate(uwline.tokens): + if index and not bracket_level: + pytree_utils.SetNodeAnnotation(token.node, + pytree_utils.Annotation.SPLIT_PENALTY, + split_penalty.UNBREAKABLE) + if token.value in pytree_utils.OPENING_BRACKETS: + bracket_level += 1 + elif token.value in pytree_utils.CLOSING_BRACKETS: + bracket_level -= 1 + + +def _DetermineMustSplitAnnotation(node): + """Enforce a split in the list if the list ends with a comma.""" + if style.Get('DISABLE_ENDING_COMMA_HEURISTIC'): + return + if not _ContainsComments(node): + token = next(node.parent.leaves()) + if token.value == '(': + if sum(1 for ch in node.children + if pytree_utils.NodeName(ch) == 'COMMA') < 2: + return + if (not isinstance(node.children[-1], pytree.Leaf) or + node.children[-1].value != ','): + return + num_children = len(node.children) + index = 0 + _SetMustSplitOnFirstLeaf(node.children[0]) + while index < num_children - 1: + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == ',': + next_child = node.children[index + 1] + if next_child.type == grammar_token.COMMENT: + index += 1 + if index >= num_children - 1: + break + _SetMustSplitOnFirstLeaf(node.children[index + 1]) + index += 1 + + +def _ContainsComments(node): + """Return True if the list has a comment in it.""" + if isinstance(node, pytree.Leaf): + return node.type == grammar_token.COMMENT + for child in node.children: + if _ContainsComments(child): + return True + return False + + +def _SetMustSplitOnFirstLeaf(node): + """Set the "must split" annotation on the first leaf node.""" + pytree_utils.SetNodeAnnotation( + pytree_utils.FirstLeafNode(node), pytree_utils.Annotation.MUST_SPLIT, + True) diff --git a/yapf/yapflib/pytree_utils.py b/yapf/yapflib/pytree_utils.py new file mode 100644 index 0000000..999ba88 --- /dev/null +++ b/yapf/yapflib/pytree_utils.py @@ -0,0 +1,334 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""pytree-related utilities. + +This module collects various utilities related to the parse trees produced by +the lib2to3 library. + + NodeName(): produces a string name for pytree nodes. + ParseCodeToTree(): convenience wrapper around lib2to3 interfaces to parse + a given string with code to a pytree. + InsertNodeBefore(): insert a node before another in a pytree. + InsertNodeAfter(): insert a node after another in a pytree. + {Get,Set}NodeAnnotation(): manage custom annotations on pytree nodes. +""" + +import ast + +from lib2to3 import pygram +from lib2to3 import pytree +from lib2to3.pgen2 import driver +from lib2to3.pgen2 import parse +from lib2to3.pgen2 import token + +# TODO(eliben): We may want to get rid of this filtering at some point once we +# have a better understanding of what information we need from the tree. Then, +# these tokens may be filtered out from the tree before the tree gets to the +# unwrapper. +NONSEMANTIC_TOKENS = frozenset(['DEDENT', 'INDENT', 'NEWLINE', 'ENDMARKER']) + +OPENING_BRACKETS = frozenset({'(', '[', '{'}) +CLOSING_BRACKETS = frozenset({')', ']', '}'}) + + +class Annotation(object): + """Annotation names associated with pytrees.""" + CHILD_INDENT = 'child_indent' + NEWLINES = 'newlines' + MUST_SPLIT = 'must_split' + SPLIT_PENALTY = 'split_penalty' + SUBTYPE = 'subtype' + + +def NodeName(node): + """Produce a string name for a given node. + + For a Leaf this is the token name, and for a Node this is the type. + + Arguments: + node: a tree node + + Returns: + Name as a string. + """ + # Nodes with values < 256 are tokens. Values >= 256 are grammar symbols. + if node.type < 256: + return token.tok_name[node.type] + else: + return pygram.python_grammar.number2symbol[node.type] + + +def FirstLeafNode(node): + if isinstance(node, pytree.Leaf): + return node + return FirstLeafNode(node.children[0]) + + +def LastLeafNode(node): + if isinstance(node, pytree.Leaf): + return node + return LastLeafNode(node.children[-1]) + + +# lib2to3 thoughtfully provides pygram.python_grammar_no_print_statement for +# parsing Python 3 code that wouldn't parse otherwise (when 'print' is used in a +# context where a keyword is disallowed). +# It forgets to do the same for 'exec' though. Luckily, Python is amenable to +# monkey-patching. +_GRAMMAR_FOR_PY3 = pygram.python_grammar_no_print_statement.copy() +del _GRAMMAR_FOR_PY3.keywords['exec'] + +_GRAMMAR_FOR_PY2 = pygram.python_grammar.copy() +del _GRAMMAR_FOR_PY2.keywords['nonlocal'] + + +def ParseCodeToTree(code): + """Parse the given code to a lib2to3 pytree. + + Arguments: + code: a string with the code to parse. + + Raises: + SyntaxError if the code is invalid syntax. + parse.ParseError if some other parsing failure. + + Returns: + The root node of the parsed tree. + """ + # This function is tiny, but the incantation for invoking the parser correctly + # is sufficiently magical to be worth abstracting away. + try: + # Try to parse using a Python 3 grammar, which is more permissive (print and + # exec are not keywords). + parser_driver = driver.Driver(_GRAMMAR_FOR_PY3, convert=pytree.convert) + tree = parser_driver.parse_string(code, debug=False) + except parse.ParseError: + # Now try to parse using a Python 2 grammar; If this fails, then + # there's something else wrong with the code. + try: + parser_driver = driver.Driver(_GRAMMAR_FOR_PY2, convert=pytree.convert) + tree = parser_driver.parse_string(code, debug=False) + except parse.ParseError: + # Raise a syntax error if the code is invalid python syntax. + try: + ast.parse(code) + except SyntaxError as e: + raise e + else: + raise + return _WrapEndMarker(tree) + + +def _WrapEndMarker(tree): + """Wrap a single ENDMARKER token in a "file_input" node. + + Arguments: + tree: (pytree.Node) The root node of the parsed tree. + + Returns: + The root node of the parsed tree. If the tree is a single ENDMARKER node, + then that node is wrapped in a "file_input" node. That will ensure we don't + skip comments attached to that node. + """ + if isinstance(tree, pytree.Leaf) and tree.type == token.ENDMARKER: + return pytree.Node(pygram.python_symbols.file_input, [tree]) + return tree + + +def InsertNodesBefore(new_nodes, target): + """Insert new_nodes before the given target location in the tree. + + Arguments: + new_nodes: a sequence of new nodes to insert (the nodes should not be in the + tree). + target: the target node before which the new node node will be inserted. + + Raises: + RuntimeError: if the tree is corrupted, or the insertion would corrupt it. + """ + for node in new_nodes: + _InsertNodeAt(node, target, after=False) + + +def InsertNodesAfter(new_nodes, target): + """Insert new_nodes after the given target location in the tree. + + Arguments: + new_nodes: a sequence of new nodes to insert (the nodes should not be in the + tree). + target: the target node after which the new node node will be inserted. + + Raises: + RuntimeError: if the tree is corrupted, or the insertion would corrupt it. + """ + for node in reversed(new_nodes): + _InsertNodeAt(node, target, after=True) + + +def _InsertNodeAt(new_node, target, after=False): + """Underlying implementation for node insertion. + + Arguments: + new_node: a new node to insert (this node should not be in the tree). + target: the target node. + after: if True, new_node is inserted after target. Otherwise, it's inserted + before target. + + Returns: + nothing + + Raises: + RuntimeError: if the tree is corrupted, or the insertion would corrupt it. + """ + + # Protect against attempts to insert nodes which already belong to some tree. + if new_node.parent is not None: + raise RuntimeError('inserting node which already has a parent', + (new_node, new_node.parent)) + + # The code here is based on pytree.Base.next_sibling + parent_of_target = target.parent + if parent_of_target is None: + raise RuntimeError('expected target node to have a parent', (target,)) + + for i, child in enumerate(parent_of_target.children): + if child is target: + insertion_index = i + 1 if after else i + parent_of_target.insert_child(insertion_index, new_node) + return + + raise RuntimeError('unable to find insertion point for target node', + (target,)) + + +# The following constant and functions implement a simple custom annotation +# mechanism for pytree nodes. We attach new attributes to nodes. Each attribute +# is prefixed with _NODE_ANNOTATION_PREFIX. These annotations should only be +# managed through GetNodeAnnotation and SetNodeAnnotation. +_NODE_ANNOTATION_PREFIX = '_yapf_annotation_' + + +def GetNodeAnnotation(node, annotation, default=None): + """Get annotation value from a node. + + Arguments: + node: the node. + annotation: annotation name - a string. + default: the default value to return if there's no annotation. + + Returns: + Value of the annotation in the given node. If the node doesn't have this + particular annotation name yet, returns default. + """ + return getattr(node, _NODE_ANNOTATION_PREFIX + annotation, default) + + +def SetNodeAnnotation(node, annotation, value): + """Set annotation value on a node. + + Arguments: + node: the node. + annotation: annotation name - a string. + value: annotation value to set. + """ + setattr(node, _NODE_ANNOTATION_PREFIX + annotation, value) + + +def AppendNodeAnnotation(node, annotation, value): + """Appends an annotation value to a list of annotations on the node. + + Arguments: + node: the node. + annotation: annotation name - a string. + value: annotation value to set. + """ + attr = GetNodeAnnotation(node, annotation, set()) + attr.add(value) + SetNodeAnnotation(node, annotation, attr) + + +def RemoveSubtypeAnnotation(node, value): + """Removes an annotation value from the subtype annotations on the node. + + Arguments: + node: the node. + value: annotation value to remove. + """ + attr = GetNodeAnnotation(node, Annotation.SUBTYPE) + if attr and value in attr: + attr.remove(value) + SetNodeAnnotation(node, Annotation.SUBTYPE, attr) + + +def GetOpeningBracket(node): + """Get opening bracket value from a node. + + Arguments: + node: the node. + + Returns: + The opening bracket node or None if it couldn't find one. + """ + return getattr(node, _NODE_ANNOTATION_PREFIX + 'container_bracket', None) + + +def SetOpeningBracket(node, bracket): + """Set opening bracket value for a node. + + Arguments: + node: the node. + bracket: opening bracket to set. + """ + setattr(node, _NODE_ANNOTATION_PREFIX + 'container_bracket', bracket) + + +def DumpNodeToString(node): + """Dump a string representation of the given node. For debugging. + + Arguments: + node: the node. + + Returns: + The string representation. + """ + if isinstance(node, pytree.Leaf): + fmt = ('{name}({value}) [lineno={lineno}, column={column}, ' + 'prefix={prefix}, penalty={penalty}]') + return fmt.format( + name=NodeName(node), + value=_PytreeNodeRepr(node), + lineno=node.lineno, + column=node.column, + prefix=repr(node.prefix), + penalty=GetNodeAnnotation(node, Annotation.SPLIT_PENALTY, None)) + else: + fmt = '{node} [{len} children] [child_indent="{indent}"]' + return fmt.format( + node=NodeName(node), + len=len(node.children), + indent=GetNodeAnnotation(node, Annotation.CHILD_INDENT)) + + +def _PytreeNodeRepr(node): + """Like pytree.Node.__repr__, but names instead of numbers for tokens.""" + if isinstance(node, pytree.Node): + return '%s(%s, %r)' % (node.__class__.__name__, NodeName(node), + [_PytreeNodeRepr(c) for c in node.children]) + if isinstance(node, pytree.Leaf): + return '%s(%s, %r)' % (node.__class__.__name__, NodeName(node), node.value) + + +def IsCommentStatement(node): + return (NodeName(node) == 'simple_stmt' and + node.children[0].type == token.COMMENT) diff --git a/yapf/yapflib/pytree_visitor.py b/yapf/yapflib/pytree_visitor.py new file mode 100644 index 0000000..49da056 --- /dev/null +++ b/yapf/yapflib/pytree_visitor.py @@ -0,0 +1,135 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generic visitor pattern for pytrees. + +The lib2to3 parser produces a "pytree" - syntax tree consisting of Node +and Leaf types. This module implements a visitor pattern for such trees. + +It also exports a basic "dumping" visitor that dumps a textual representation of +a pytree into a stream. + + PyTreeVisitor: a generic visitor pattern fo pytrees. + PyTreeDumper: a configurable "dumper" for displaying pytrees. + DumpPyTree(): a convenience function to dump a pytree. +""" + +import sys + +from lib2to3 import pytree + +from yapf.yapflib import pytree_utils + + +class PyTreeVisitor(object): + """Visitor pattern for pytree trees. + + Methods named Visit_XXX will be invoked when a node with type XXX is + encountered in the tree. The type is either a token type (for Leaf nodes) or + grammar symbols (for Node nodes). The return value of Visit_XXX methods is + ignored by the visitor. + + Visitors can modify node contents but must not change the tree structure + (e.g. add/remove children and move nodes around). + + This is a very common visitor pattern in Python code; it's also used in the + Python standard library ast module for providing AST visitors. + + Note: this makes names that aren't style conformant, so such visitor methods + need to be marked with # pylint: disable=invalid-name We don't have a choice + here, because lib2to3 nodes have under_separated names. + + For more complex behavior, the visit, DefaultNodeVisit and DefaultLeafVisit + methods can be overridden. Don't forget to invoke DefaultNodeVisit for nodes + that may have children - otherwise the children will not be visited. + """ + + def Visit(self, node): + """Visit a node.""" + method = 'Visit_{0}'.format(pytree_utils.NodeName(node)) + if hasattr(self, method): + # Found a specific visitor for this node + getattr(self, method)(node) + else: + if isinstance(node, pytree.Leaf): + self.DefaultLeafVisit(node) + else: + self.DefaultNodeVisit(node) + + def DefaultNodeVisit(self, node): + """Default visitor for Node: visits the node's children depth-first. + + This method is invoked when no specific visitor for the node is defined. + + Arguments: + node: the node to visit + """ + for child in node.children: + self.Visit(child) + + def DefaultLeafVisit(self, leaf): + """Default visitor for Leaf: no-op. + + This method is invoked when no specific visitor for the leaf is defined. + + Arguments: + leaf: the leaf to visit + """ + pass + + +def DumpPyTree(tree, target_stream=sys.stdout): + """Convenience function for dumping a given pytree. + + This function presents a very minimal interface. For more configurability (for + example, controlling how specific node types are displayed), use PyTreeDumper + directly. + + Arguments: + tree: the tree to dump. + target_stream: the stream to dump the tree to. A file-like object. By + default will dump into stdout. + """ + dumper = PyTreeDumper(target_stream) + dumper.Visit(tree) + + +class PyTreeDumper(PyTreeVisitor): + """Visitor that dumps the tree to a stream. + + Implements the PyTreeVisitor interface. + """ + + def __init__(self, target_stream=sys.stdout): + """Create a tree dumper. + + Arguments: + target_stream: the stream to dump the tree to. A file-like object. By + default will dump into stdout. + """ + self._target_stream = target_stream + self._current_indent = 0 + + def _DumpString(self, s): + self._target_stream.write('{0}{1}\n'.format(' ' * self._current_indent, s)) + + def DefaultNodeVisit(self, node): + # Dump information about the current node, and then use the generic + # DefaultNodeVisit visitor to dump each of its children. + self._DumpString(pytree_utils.DumpNodeToString(node)) + self._current_indent += 2 + super(PyTreeDumper, self).DefaultNodeVisit(node) + self._current_indent -= 2 + + def DefaultLeafVisit(self, leaf): + self._DumpString(pytree_utils.DumpNodeToString(leaf)) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py new file mode 100644 index 0000000..6539e68 --- /dev/null +++ b/yapf/yapflib/reformatter.py @@ -0,0 +1,617 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Decide what the format for the code should be. + +The `unwrapped_line.UnwrappedLine`s are now ready to be formatted. +UnwrappedLines that can be merged together are. The best formatting is returned +as a string. + + Reformat(): the main function exported by this module. +""" + +from __future__ import unicode_literals +import collections +import heapq +import re + +from lib2to3 import pytree +from lib2to3.pgen2 import token + +from yapf.yapflib import format_decision_state +from yapf.yapflib import format_token +from yapf.yapflib import line_joiner +from yapf.yapflib import pytree_utils +from yapf.yapflib import style +from yapf.yapflib import verifier + + +def Reformat(uwlines, verify=False, lines=None): + """Reformat the unwrapped lines. + + Arguments: + uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format. + verify: (bool) True if reformatted code should be verified for syntax. + lines: (set of int) The lines which can be modified or None if there is no + line range restriction. + + Returns: + A string representing the reformatted code. + """ + final_lines = [] + prev_uwline = None # The previous line. + indent_width = style.Get('INDENT_WIDTH') + + for uwline in _SingleOrMergedLines(uwlines): + first_token = uwline.first + _FormatFirstToken(first_token, uwline.depth, prev_uwline, final_lines) + + indent_amt = indent_width * uwline.depth + state = format_decision_state.FormatDecisionState(uwline, indent_amt) + state.MoveStateToNextToken() + + if not uwline.disable: + if uwline.first.is_comment: + uwline.first.node.value = uwline.first.node.value.rstrip() + elif uwline.last.is_comment: + uwline.last.node.value = uwline.last.node.value.rstrip() + if prev_uwline and prev_uwline.disable: + # Keep the vertical spacing between a disabled and enabled formatting + # region. + _RetainRequiredVerticalSpacingBetweenTokens(uwline.first, + prev_uwline.last, lines) + if any(tok.is_comment for tok in uwline.tokens): + _RetainVerticalSpacingBeforeComments(uwline) + + if (_LineContainsI18n(uwline) or uwline.disable or + _LineHasContinuationMarkers(uwline)): + _RetainHorizontalSpacing(uwline) + _RetainRequiredVerticalSpacing(uwline, prev_uwline, lines) + _EmitLineUnformatted(state) + elif _CanPlaceOnSingleLine(uwline) and not any(tok.must_split + for tok in uwline.tokens): + # The unwrapped line fits on one line. + while state.next_token: + state.AddTokenToState(newline=False, dry_run=False) + else: + if not _AnalyzeSolutionSpace(state): + # Failsafe mode. If there isn't a solution to the line, then just emit + # it as is. + state = format_decision_state.FormatDecisionState(uwline, indent_amt) + state.MoveStateToNextToken() + _RetainHorizontalSpacing(uwline) + _RetainRequiredVerticalSpacing(uwline, prev_uwline, None) + _EmitLineUnformatted(state) + + final_lines.append(uwline) + prev_uwline = uwline + return _FormatFinalLines(final_lines, verify) + + +def _RetainHorizontalSpacing(uwline): + """Retain all horizontal spacing between tokens.""" + for tok in uwline.tokens: + tok.RetainHorizontalSpacing(uwline.first.column, uwline.depth) + + +def _RetainRequiredVerticalSpacing(cur_uwline, prev_uwline, lines): + prev_tok = None + if prev_uwline is not None: + prev_tok = prev_uwline.last + for cur_tok in cur_uwline.tokens: + _RetainRequiredVerticalSpacingBetweenTokens(cur_tok, prev_tok, lines) + prev_tok = cur_tok + + +def _RetainRequiredVerticalSpacingBetweenTokens(cur_tok, prev_tok, lines): + """Retain vertical spacing between two tokens if not in editable range.""" + if prev_tok is None: + return + + if prev_tok.is_string: + prev_lineno = prev_tok.lineno + prev_tok.value.count('\n') + elif prev_tok.is_pseudo_paren: + if not prev_tok.previous_token.is_multiline_string: + prev_lineno = prev_tok.previous_token.lineno + else: + prev_lineno = prev_tok.lineno + else: + prev_lineno = prev_tok.lineno + + if cur_tok.is_comment: + cur_lineno = cur_tok.lineno - cur_tok.value.count('\n') + else: + cur_lineno = cur_tok.lineno + + if prev_tok.value.endswith('\\'): + prev_lineno += prev_tok.value.count('\n') + + required_newlines = cur_lineno - prev_lineno + if cur_tok.is_comment and not prev_tok.is_comment: + # Don't adjust between a comment and non-comment. + pass + elif lines and (cur_lineno in lines or prev_lineno in lines): + desired_newlines = cur_tok.whitespace_prefix.count('\n') + whitespace_lines = range(prev_lineno + 1, cur_lineno) + deletable_lines = len(lines.intersection(whitespace_lines)) + required_newlines = max(required_newlines - deletable_lines, + desired_newlines) + + cur_tok.AdjustNewlinesBefore(required_newlines) + + +def _RetainVerticalSpacingBeforeComments(uwline): + """Retain vertical spacing before comments.""" + prev_token = None + for tok in uwline.tokens: + if tok.is_comment and prev_token: + if tok.lineno - tok.value.count('\n') - prev_token.lineno > 1: + tok.AdjustNewlinesBefore(ONE_BLANK_LINE) + + prev_token = tok + + +def _EmitLineUnformatted(state): + """Emit the line without formatting. + + The line contains code that if reformatted would break a non-syntactic + convention. E.g., i18n comments and function calls are tightly bound by + convention. Instead, we calculate when / if a newline should occur and honor + that. But otherwise the code emitted will be the same as the original code. + + Arguments: + state: (format_decision_state.FormatDecisionState) The format decision + state. + """ + prev_lineno = None + while state.next_token: + previous_token = state.next_token.previous_token + previous_lineno = previous_token.lineno + + if previous_token.is_multiline_string or previous_token.is_string: + previous_lineno += previous_token.value.count('\n') + + if previous_token.is_continuation: + newline = False + else: + newline = ( + prev_lineno is not None and state.next_token.lineno > previous_lineno) + + prev_lineno = state.next_token.lineno + state.AddTokenToState(newline=newline, dry_run=False) + + +def _LineContainsI18n(uwline): + """Return true if there are i18n comments or function calls in the line. + + I18n comments and pseudo-function calls are closely related. They cannot + be moved apart without breaking i18n. + + Arguments: + uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted. + + Returns: + True if the line contains i18n comments or function calls. False otherwise. + """ + if style.Get('I18N_COMMENT'): + for tok in uwline.tokens: + if tok.is_comment and re.match(style.Get('I18N_COMMENT'), tok.value): + # Contains an i18n comment. + return True + + if style.Get('I18N_FUNCTION_CALL'): + length = len(uwline.tokens) + index = 0 + while index < length - 1: + if (uwline.tokens[index + 1].value == '(' and + uwline.tokens[index].value in style.Get('I18N_FUNCTION_CALL')): + return True + index += 1 + + return False + + +def _LineHasContinuationMarkers(uwline): + """Return true if the line has continuation markers in it.""" + return any(tok.is_continuation for tok in uwline.tokens) + + +def _CanPlaceOnSingleLine(uwline): + """Determine if the unwrapped line can go on a single line. + + Arguments: + uwline: (unwrapped_line.UnwrappedLine) The line currently being formatted. + + Returns: + True if the line can or should be added to a single line. False otherwise. + """ + indent_amt = style.Get('INDENT_WIDTH') * uwline.depth + last = uwline.last + last_index = -1 + if last.is_pylint_comment or last.is_pytype_comment: + last = last.previous_token + last_index = -2 + if last is None: + return True + return (last.total_length + indent_amt <= style.Get('COLUMN_LIMIT') and + not any(tok.is_comment for tok in uwline.tokens[:last_index])) + + +def _FormatFinalLines(final_lines, verify): + """Compose the final output from the finalized lines.""" + formatted_code = [] + for line in final_lines: + formatted_line = [] + for tok in line.tokens: + if not tok.is_pseudo_paren: + formatted_line.append(tok.whitespace_prefix) + formatted_line.append(tok.value) + else: + if (not tok.next_token.whitespace_prefix.startswith('\n') and + not tok.next_token.whitespace_prefix.startswith(' ')): + if (tok.previous_token.value == ':' or + tok.next_token.value not in ',}])'): + formatted_line.append(' ') + + formatted_code.append(''.join(formatted_line)) + if verify: + verifier.VerifyCode(formatted_code[-1]) + + return ''.join(formatted_code) + '\n' + + +class _StateNode(object): + """An edge in the solution space from 'previous.state' to 'state'. + + Attributes: + state: (format_decision_state.FormatDecisionState) The format decision state + for this node. + newline: If True, then on the edge from 'previous.state' to 'state' a + newline is inserted. + previous: (_StateNode) The previous state node in the graph. + """ + + # TODO(morbo): Add a '__cmp__' method. + + def __init__(self, state, newline, previous): + self.state = state.Clone() + self.newline = newline + self.previous = previous + + def __repr__(self): # pragma: no cover + return 'StateNode(state=[\n{0}\n], newline={1})'.format( + self.state, self.newline) + + +# A tuple of (penalty, count) that is used to prioritize the BFS. In case of +# equal penalties, we prefer states that were inserted first. During state +# generation, we make sure that we insert states first that break the line as +# late as possible. +_OrderedPenalty = collections.namedtuple('OrderedPenalty', ['penalty', 'count']) + +# An item in the prioritized BFS search queue. The 'StateNode's 'state' has +# the given '_OrderedPenalty'. +_QueueItem = collections.namedtuple('QueueItem', + ['ordered_penalty', 'state_node']) + + +def _AnalyzeSolutionSpace(initial_state): + """Analyze the entire solution space starting from initial_state. + + This implements a variant of Dijkstra's algorithm on the graph that spans + the solution space (LineStates are the nodes). The algorithm tries to find + the shortest path (the one with the lowest penalty) from 'initial_state' to + the state where all tokens are placed. + + Arguments: + initial_state: (format_decision_state.FormatDecisionState) The initial state + to start the search from. + + Returns: + True if a formatting solution was found. False otherwise. + """ + count = 0 + seen = set() + p_queue = [] + + # Insert start element. + node = _StateNode(initial_state, False, None) + heapq.heappush(p_queue, _QueueItem(_OrderedPenalty(0, count), node)) + + count += 1 + while p_queue: + item = p_queue[0] + penalty = item.ordered_penalty.penalty + node = item.state_node + if not node.state.next_token: + break + heapq.heappop(p_queue) + + if count > 10000: + node.state.ignore_stack_for_comparison = True + + if node.state in seen: + continue + + seen.add(node.state) + + # FIXME(morbo): Add a 'decision' element? + + count = _AddNextStateToQueue(penalty, node, False, count, p_queue) + count = _AddNextStateToQueue(penalty, node, True, count, p_queue) + + if not p_queue: + # We weren't able to find a solution. Do nothing. + return False + + _ReconstructPath(initial_state, heapq.heappop(p_queue).state_node) + return True + + +def _AddNextStateToQueue(penalty, previous_node, newline, count, p_queue): + """Add the following state to the analysis queue. + + Assume the current state is 'previous_node' and has been reached with a + penalty of 'penalty'. Insert a line break if 'newline' is True. + + Arguments: + penalty: (int) The penalty associated with the path up to this point. + previous_node: (_StateNode) The last _StateNode inserted into the priority + queue. + newline: (bool) Add a newline if True. + count: (int) The number of elements in the queue. + p_queue: (heapq) The priority queue representing the solution space. + + Returns: + The updated number of elements in the queue. + """ + must_split = previous_node.state.MustSplit() + if newline and not previous_node.state.CanSplit(must_split): + # Don't add a newline if the token cannot be split. + return count + if not newline and must_split: + # Don't add a token we must split but where we aren't splitting. + return count + + node = _StateNode(previous_node.state, newline, previous_node) + penalty += node.state.AddTokenToState( + newline=newline, dry_run=True, must_split=must_split) + heapq.heappush(p_queue, _QueueItem(_OrderedPenalty(penalty, count), node)) + return count + 1 + + +def _ReconstructPath(initial_state, current): + """Reconstruct the path through the queue with lowest penalty. + + Arguments: + initial_state: (format_decision_state.FormatDecisionState) The initial state + to start the search from. + current: (_StateNode) The node in the decision graph that is the end point + of the path with the least penalty. + """ + path = collections.deque() + + while current.previous: + path.appendleft(current) + current = current.previous + + for node in path: + initial_state.AddTokenToState(newline=node.newline, dry_run=False) + + +def _FormatFirstToken(first_token, indent_depth, prev_uwline, final_lines): + """Format the first token in the unwrapped line. + + Add a newline and the required indent before the first token of the unwrapped + line. + + Arguments: + first_token: (format_token.FormatToken) The first token in the unwrapped + line. + indent_depth: (int) The line's indentation depth. + prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line + previous to this line. + final_lines: (list of unwrapped_line.UnwrappedLine) The unwrapped lines + that have already been processed. + """ + first_token.AddWhitespacePrefix( + _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, + final_lines), + indent_level=indent_depth) + + +NO_BLANK_LINES = 1 +ONE_BLANK_LINE = 2 +TWO_BLANK_LINES = 3 + + +def _IsClassOrDef(uwline): + if uwline.first.value in {'class', 'def'}: + return True + + return [t.value for t in uwline.tokens[:2]] == ['async', 'def'] + + +def _CalculateNumberOfNewlines(first_token, indent_depth, prev_uwline, + final_lines): + """Calculate the number of newlines we need to add. + + Arguments: + first_token: (format_token.FormatToken) The first token in the unwrapped + line. + indent_depth: (int) The line's indentation depth. + prev_uwline: (list of unwrapped_line.UnwrappedLine) The unwrapped line + previous to this line. + final_lines: (list of unwrapped_line.UnwrappedLine) The unwrapped lines + that have already been processed. + + Returns: + The number of newlines needed before the first token. + """ + # TODO(morbo): Special handling for imports. + # TODO(morbo): Create a knob that can tune these. + if prev_uwline is None: + # The first line in the file. Don't add blank lines. + # FIXME(morbo): Is this correct? + if first_token.newlines is not None: + pytree_utils.SetNodeAnnotation(first_token.node, + pytree_utils.Annotation.NEWLINES, None) + return 0 + + if first_token.is_docstring: + if (prev_uwline.first.value == 'class' and + style.Get('BLANK_LINE_BEFORE_CLASS_DOCSTRING')): + # Enforce a blank line before a class's docstring. + return ONE_BLANK_LINE + elif (prev_uwline.first.value.startswith('#') and + style.Get('BLANK_LINE_BEFORE_MODULE_DOCSTRING')): + # Enforce a blank line before a module's docstring. + return ONE_BLANK_LINE + # The docstring shouldn't have a newline before it. + return NO_BLANK_LINES + + prev_last_token = prev_uwline.last + if prev_last_token.is_docstring: + if (not indent_depth and first_token.value in {'class', 'def', 'async'}): + # Separate a class or function from the module-level docstring with + # appropriate number of blank lines. + return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION') + if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token, + prev_last_token): + return NO_BLANK_LINES + else: + return ONE_BLANK_LINE + + if first_token.value in {'class', 'def', 'async', '@'}: + # TODO(morbo): This can go once the blank line calculator is more + # sophisticated. + if not indent_depth: + # This is a top-level class or function. + is_inline_comment = prev_last_token.whitespace_prefix.count('\n') == 0 + if (not prev_uwline.disable and prev_last_token.is_comment and + not is_inline_comment): + # This token follows a non-inline comment. + if _NoBlankLinesBeforeCurrentToken(prev_last_token.value, first_token, + prev_last_token): + # Assume that the comment is "attached" to the current line. + # Therefore, we want two blank lines before the comment. + index = len(final_lines) - 1 + while index > 0: + if not final_lines[index - 1].is_comment: + break + index -= 1 + if final_lines[index - 1].first.value == '@': + final_lines[index].first.AdjustNewlinesBefore(NO_BLANK_LINES) + else: + prev_last_token.AdjustNewlinesBefore( + 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION')) + if first_token.newlines is not None: + pytree_utils.SetNodeAnnotation( + first_token.node, pytree_utils.Annotation.NEWLINES, None) + return NO_BLANK_LINES + elif _IsClassOrDef(prev_uwline): + if not style.Get('BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'): + pytree_utils.SetNodeAnnotation(first_token.node, + pytree_utils.Annotation.NEWLINES, None) + return NO_BLANK_LINES + + # Calculate how many newlines were between the original lines. We want to + # retain that formatting if it doesn't violate one of the style guide rules. + if first_token.is_comment: + first_token_lineno = first_token.lineno - first_token.value.count('\n') + else: + first_token_lineno = first_token.lineno + + prev_last_token_lineno = prev_last_token.lineno + if prev_last_token.is_multiline_string: + prev_last_token_lineno += prev_last_token.value.count('\n') + + if first_token_lineno - prev_last_token_lineno > 1: + return ONE_BLANK_LINE + + return NO_BLANK_LINES + + +def _SingleOrMergedLines(uwlines): + """Generate the lines we want to format. + + Arguments: + uwlines: (list of unwrapped_line.UnwrappedLine) Lines we want to format. + + Yields: + Either a single line, if the current line cannot be merged with the + succeeding line, or the next two lines merged into one line. + """ + index = 0 + last_was_merged = False + while index < len(uwlines): + if uwlines[index].disable: + uwline = uwlines[index] + index += 1 + while index < len(uwlines): + column = uwline.last.column + 2 + if uwlines[index].lineno != uwline.lineno: + break + if uwline.last.value != ':': + leaf = pytree.Leaf( + type=token.SEMI, value=';', context=('', (uwline.lineno, column))) + uwline.AppendToken(format_token.FormatToken(leaf)) + for tok in uwlines[index].tokens: + uwline.AppendToken(tok) + index += 1 + yield uwline + elif line_joiner.CanMergeMultipleLines(uwlines[index:], last_was_merged): + # TODO(morbo): This splice is potentially very slow. Come up with a more + # performance-friendly way of determining if two lines can be merged. + next_uwline = uwlines[index + 1] + for tok in next_uwline.tokens: + uwlines[index].AppendToken(tok) + if (len(next_uwline.tokens) == 1 and + next_uwline.first.is_multiline_string): + # This may be a multiline shebang. In that case, we want to retain the + # formatting. Otherwise, it could mess up the shell script's syntax. + uwlines[index].disable = True + yield uwlines[index] + index += 2 + last_was_merged = True + else: + yield uwlines[index] + index += 1 + last_was_merged = False + + +def _NoBlankLinesBeforeCurrentToken(text, cur_token, prev_token): + """Determine if there are no blank lines before the current token. + + The previous token is a docstring or comment. The prev_token_lineno is the + start of the text of that token. Counting the number of newlines in its text + gives us the extent and thus where the line number of the end of the + docstring or comment. After that, we just compare it to the current token's + line number to see if there are blank lines between them. + + Arguments: + text: (unicode) The text of the docstring or comment before the current + token. + cur_token: (format_token.FormatToken) The current token in the unwrapped + line. + prev_token: (format_token.FormatToken) The previous token in the unwrapped + line. + + Returns: + True if there is no blank line before the current token. + """ + cur_token_lineno = cur_token.lineno + if cur_token.is_comment: + cur_token_lineno -= cur_token.value.count('\n') + num_newlines = text.count('\n') if not prev_token.is_comment else 0 + return prev_token.lineno + num_newlines == cur_token_lineno - 1 diff --git a/yapf/yapflib/split_penalty.py b/yapf/yapflib/split_penalty.py new file mode 100644 index 0000000..416eda3 --- /dev/null +++ b/yapf/yapflib/split_penalty.py @@ -0,0 +1,612 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Computation of split penalties before/between tokens.""" + +import re + +from lib2to3 import pytree + +from yapf.yapflib import format_token +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import style + +# TODO(morbo): Document the annotations in a centralized place. E.g., the +# README file. +UNBREAKABLE = 1000 * 1000 +NAMED_ASSIGN = 11000 +DOTTED_NAME = 4000 +VERY_STRONGLY_CONNECTED = 3500 +STRONGLY_CONNECTED = 3000 +CONNECTED = 500 + +OR_TEST = 1000 +AND_TEST = 1100 +NOT_TEST = 1200 +COMPARISON = 1300 +STAR_EXPR = 1300 +EXPR = 1400 +XOR_EXPR = 1500 +AND_EXPR = 1700 +SHIFT_EXPR = 1800 +ARITH_EXPR = 1900 +TERM = 2000 +FACTOR = 2100 +POWER = 2200 +ATOM = 2300 +ONE_ELEMENT_ARGUMENT = 2500 + + +def ComputeSplitPenalties(tree): + """Compute split penalties on tokens in the given parse tree. + + Arguments: + tree: the top-level pytree node to annotate with penalties. + """ + _SplitPenaltyAssigner().Visit(tree) + + +class _SplitPenaltyAssigner(pytree_visitor.PyTreeVisitor): + """Assigns split penalties to tokens, based on parse tree structure. + + Split penalties are attached as annotations to tokens. + """ + + def Visit_import_as_names(self, node): # pyline: disable=invalid-name + # import_as_names ::= import_as_name (',' import_as_name)* [','] + self.DefaultNodeVisit(node) + prev_child = None + for child in node.children: + if (prev_child and isinstance(prev_child, pytree.Leaf) and + prev_child.value == ','): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_IMPORT_NAMES')) + prev_child = child + + def Visit_classdef(self, node): # pylint: disable=invalid-name + # classdef ::= 'class' NAME ['(' [arglist] ')'] ':' suite + # + # NAME + _SetUnbreakable(node.children[1]) + if len(node.children) > 4: + # opening '(' + _SetUnbreakable(node.children[2]) + # ':' + _SetUnbreakable(node.children[-2]) + self.DefaultNodeVisit(node) + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + # funcdef ::= 'def' NAME parameters ['->' test] ':' suite + # + # Can't break before the function name and before the colon. The parameters + # are handled by child iteration. + colon_idx = 1 + while pytree_utils.NodeName(node.children[colon_idx]) == 'simple_stmt': + colon_idx += 1 + _SetUnbreakable(node.children[colon_idx]) + arrow_idx = -1 + while colon_idx < len(node.children): + if isinstance(node.children[colon_idx], pytree.Leaf): + if node.children[colon_idx].value == ':': + break + if node.children[colon_idx].value == '->': + arrow_idx = colon_idx + colon_idx += 1 + _SetUnbreakable(node.children[colon_idx]) + self.DefaultNodeVisit(node) + if arrow_idx > 0: + _SetSplitPenalty( + pytree_utils.LastLeafNode(node.children[arrow_idx - 1]), 0) + _SetUnbreakable(node.children[arrow_idx]) + _SetStronglyConnected(node.children[arrow_idx + 1]) + + def Visit_lambdef(self, node): # pylint: disable=invalid-name + # lambdef ::= 'lambda' [varargslist] ':' test + # Loop over the lambda up to and including the colon. + allow_multiline_lambdas = style.Get('ALLOW_MULTILINE_LAMBDAS') + if not allow_multiline_lambdas: + for child in node.children: + if pytree_utils.NodeName(child) == 'COMMENT': + if re.search(r'pylint:.*disable=.*\bg-long-lambda', child.value): + allow_multiline_lambdas = True + break + + if allow_multiline_lambdas: + _SetStronglyConnected(node) + else: + self._SetUnbreakableOnChildren(node) + + def Visit_parameters(self, node): # pylint: disable=invalid-name + # parameters ::= '(' [typedargslist] ')' + self.DefaultNodeVisit(node) + + # Can't break before the opening paren of a parameter list. + _SetUnbreakable(node.children[0]) + if not style.Get('DEDENT_CLOSING_BRACKETS'): + _SetStronglyConnected(node.children[-1]) + + def Visit_arglist(self, node): # pylint: disable=invalid-name + # arglist ::= argument (',' argument)* [','] + self.DefaultNodeVisit(node) + index = 1 + while index < len(node.children): + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == ',': + _SetUnbreakable(child) + index += 1 + for child in node.children: + if pytree_utils.NodeName(child) == 'atom': + _IncreasePenalty(child, CONNECTED) + + def Visit_argument(self, node): # pylint: disable=invalid-name + # argument ::= test [comp_for] | test '=' test # Really [keyword '='] test + self.DefaultNodeVisit(node) + index = 1 + while index < len(node.children) - 1: + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == '=': + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) + index += 1 + + def Visit_tname(self, node): # pylint: disable=invalid-name + # tname ::= NAME [':' test] + self.DefaultNodeVisit(node) + index = 1 + while index < len(node.children) - 1: + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == ':': + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), NAMED_ASSIGN) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), NAMED_ASSIGN) + index += 1 + + def Visit_dotted_name(self, node): # pylint: disable=invalid-name + # dotted_name ::= NAME ('.' NAME)* + self._SetUnbreakableOnChildren(node) + + def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name + # dictsetmaker ::= ( (test ':' test + # (comp_for | (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) ) + for child in node.children: + self.Visit(child) + if pytree_utils.NodeName(child) == 'COLON': + # This is a key to a dictionary. We don't want to split the key if at + # all possible. + _SetStronglyConnected(child) + + def Visit_trailer(self, node): # pylint: disable=invalid-name + # trailer ::= '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME + if node.children[0].value == '.': + self._SetUnbreakableOnChildren(node) + _SetSplitPenalty(node.children[1], DOTTED_NAME) + elif len(node.children) == 2: + # Don't split an empty argument list if at all possible. + _SetSplitPenalty(node.children[1], VERY_STRONGLY_CONNECTED) + elif len(node.children) == 3: + name = pytree_utils.NodeName(node.children[1]) + if name in {'argument', 'comparison'}: + # Don't split an argument list with one element if at all possible. + _SetStronglyConnected(node.children[1]) + if (len(node.children[1].children) > 1 and + pytree_utils.NodeName(node.children[1].children[1]) == 'comp_for'): + # Don't penalize splitting before a comp_for expression. + _SetSplitPenalty(pytree_utils.FirstLeafNode(node.children[1]), 0) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[1]), + ONE_ELEMENT_ARGUMENT) + elif (pytree_utils.NodeName(node.children[0]) == 'LSQB' and + len(node.children[1].children) > 2 and + (name.endswith('_test') or name.endswith('_expr'))): + _SetStronglyConnected(node.children[1].children[0]) + _SetStronglyConnected(node.children[1].children[2]) + + # Still allow splitting around the operator. + split_before = ((name.endswith('_test') and + style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR')) or + (name.endswith('_expr') and + style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'))) + if split_before: + _SetSplitPenalty( + pytree_utils.LastLeafNode(node.children[1].children[1]), 0) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[1].children[2]), 0) + + # Don't split the ending bracket of a subscript list. + _SetVeryStronglyConnected(node.children[-1]) + elif name not in { + 'arglist', 'argument', 'term', 'or_test', 'and_test', 'comparison', + 'atom', 'power' + }: + # Don't split an argument list with one element if at all possible. + _SetStronglyConnected(node.children[1], node.children[2]) + + if name == 'arglist': + _SetStronglyConnected(node.children[-1]) + + self.DefaultNodeVisit(node) + + def Visit_power(self, node): # pylint: disable=invalid-name,missing-docstring + # power ::= atom trailer* ['**' factor] + self.DefaultNodeVisit(node) + + # When atom is followed by a trailer, we can not break between them. + # E.g. arr[idx] - no break allowed between 'arr' and '['. + if (len(node.children) > 1 and + pytree_utils.NodeName(node.children[1]) == 'trailer'): + # children[1] itself is a whole trailer: we don't want to + # mark all of it as unbreakable, only its first token: (, [ or . + _SetUnbreakable(node.children[1].children[0]) + + # A special case when there are more trailers in the sequence. Given: + # atom tr1 tr2 + # The last token of tr1 and the first token of tr2 comprise an unbreakable + # region. For example: foo.bar.baz(1) + # We can't put breaks between either of the '.', '(', or '[' and the names + # *preceding* them. + prev_trailer_idx = 1 + while prev_trailer_idx < len(node.children) - 1: + cur_trailer_idx = prev_trailer_idx + 1 + cur_trailer = node.children[cur_trailer_idx] + if pytree_utils.NodeName(cur_trailer) == 'trailer': + # Now we know we have two trailers one after the other + prev_trailer = node.children[prev_trailer_idx] + if prev_trailer.children[-1].value != ')': + # Set the previous node unbreakable if it's not a function call: + # atom tr1() tr2 + # It may be necessary (though undesirable) to split up a previous + # function call's parentheses to the next line. + _SetStronglyConnected(prev_trailer.children[-1]) + _SetStronglyConnected(cur_trailer.children[0]) + prev_trailer_idx = cur_trailer_idx + else: + break + + # We don't want to split before the last ')' of a function call. This also + # takes care of the special case of: + # atom tr1 tr2 ... trn + # where the 'tr#' are trailers that may end in a ')'. + for trailer in node.children[1:]: + if pytree_utils.NodeName(trailer) != 'trailer': + break + if trailer.children[0].value in '([': + if len(trailer.children) > 2: + subtypes = pytree_utils.GetNodeAnnotation( + trailer.children[0], pytree_utils.Annotation.SUBTYPE) + if subtypes and format_token.Subtype.SUBSCRIPT_BRACKET in subtypes: + _SetStronglyConnected( + pytree_utils.FirstLeafNode(trailer.children[1])) + + last_child_node = pytree_utils.LastLeafNode(trailer) + if last_child_node.value.strip().startswith('#'): + last_child_node = last_child_node.prev_sibling + if not style.Get('DEDENT_CLOSING_BRACKETS'): + last = pytree_utils.LastLeafNode(last_child_node.prev_sibling) + if last.value != ',': + if last_child_node.value == ']': + _SetUnbreakable(last_child_node) + else: + _SetSplitPenalty(last_child_node, VERY_STRONGLY_CONNECTED) + else: + # If the trailer's children are '()', then make it a strongly + # connected region. It's sometimes necessary, though undesirable, to + # split the two. + _SetStronglyConnected(trailer.children[-1]) + + # If the original source has a "builder" style calls, then we should allow + # the reformatter to retain that. + _AllowBuilderStyleCalls(node) + + def Visit_subscript(self, node): # pylint: disable=invalid-name + # subscript ::= test | [test] ':' [test] [sliceop] + _SetStronglyConnected(*node.children) + self.DefaultNodeVisit(node) + + def Visit_comp_for(self, node): # pylint: disable=invalid-name + # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] + _SetSplitPenalty(pytree_utils.FirstLeafNode(node), 0) + _SetStronglyConnected(*node.children[1:]) + self.DefaultNodeVisit(node) + + def Visit_comp_if(self, node): # pylint: disable=invalid-name + # comp_if ::= 'if' old_test [comp_iter] + _SetSplitPenalty(node.children[0], + style.Get('SPLIT_PENALTY_BEFORE_IF_EXPR')) + _SetStronglyConnected(*node.children[1:]) + self.DefaultNodeVisit(node) + + def Visit_or_test(self, node): # pylint: disable=invalid-name + # or_test ::= and_test ('or' and_test)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, OR_TEST) + index = 1 + while index + 1 < len(node.children): + if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'): + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), OR_TEST) + else: + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), OR_TEST) + index += 2 + + def Visit_and_test(self, node): # pylint: disable=invalid-name + # and_test ::= not_test ('and' not_test)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, AND_TEST) + index = 1 + while index + 1 < len(node.children): + if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'): + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index]), AND_TEST) + else: + _DecrementSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), AND_TEST) + index += 2 + + def Visit_not_test(self, node): # pylint: disable=invalid-name + # not_test ::= 'not' not_test | comparison + self.DefaultNodeVisit(node) + _IncreasePenalty(node, NOT_TEST) + + def Visit_comparison(self, node): # pylint: disable=invalid-name + # comparison ::= expr (comp_op expr)* + self.DefaultNodeVisit(node) + if len(node.children) == 3 and _StronglyConnectedCompOp(node): + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[1]), STRONGLY_CONNECTED) + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[2]), STRONGLY_CONNECTED) + else: + _IncreasePenalty(node, COMPARISON) + + def Visit_star_expr(self, node): # pylint: disable=invalid-name + # star_expr ::= '*' expr + self.DefaultNodeVisit(node) + _IncreasePenalty(node, STAR_EXPR) + + def Visit_expr(self, node): # pylint: disable=invalid-name + # expr ::= xor_expr ('|' xor_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, EXPR) + index = 1 + while index < len(node.children) - 1: + child = node.children[index] + if isinstance(child, pytree.Leaf) and child.value == '|': + if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'): + _SetSplitPenalty(child, style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + else: + _SetSplitPenalty( + pytree_utils.FirstLeafNode(node.children[index + 1]), + style.Get('SPLIT_PENALTY_BITWISE_OPERATOR')) + index += 1 + + def Visit_xor_expr(self, node): # pylint: disable=invalid-name + # xor_expr ::= and_expr ('^' and_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, XOR_EXPR) + + def Visit_and_expr(self, node): # pylint: disable=invalid-name + # and_expr ::= shift_expr ('&' shift_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, AND_EXPR) + + def Visit_shift_expr(self, node): # pylint: disable=invalid-name + # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, SHIFT_EXPR) + + _ARITH_OPS = frozenset({'PLUS', 'MINUS'}) + + def Visit_arith_expr(self, node): # pylint: disable=invalid-name + # arith_expr ::= term (('+'|'-') term)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, ARITH_EXPR) + + index = 1 + while index < len(node.children) - 1: + child = node.children[index] + if pytree_utils.NodeName(child) in self._ARITH_OPS: + next_node = pytree_utils.FirstLeafNode(node.children[index + 1]) + _SetSplitPenalty(next_node, ARITH_EXPR) + index += 1 + + _TERM_OPS = frozenset({'STAR', 'AT', 'SLASH', 'PERCENT', 'DOUBLESLASH'}) + + def Visit_term(self, node): # pylint: disable=invalid-name + # term ::= factor (('*'|'@'|'/'|'%'|'//') factor)* + self.DefaultNodeVisit(node) + _IncreasePenalty(node, TERM) + + index = 1 + while index < len(node.children) - 1: + child = node.children[index] + if pytree_utils.NodeName(child) in self._TERM_OPS: + next_node = pytree_utils.FirstLeafNode(node.children[index + 1]) + _SetSplitPenalty(next_node, TERM) + index += 1 + + def Visit_factor(self, node): # pyline: disable=invalid-name + # factor ::= ('+'|'-'|'~') factor | power + self.DefaultNodeVisit(node) + _IncreasePenalty(node, FACTOR) + + def Visit_atom(self, node): # pylint: disable=invalid-name + # atom ::= ('(' [yield_expr|testlist_gexp] ')' + # '[' [listmaker] ']' | + # '{' [dictsetmaker] '}') + self.DefaultNodeVisit(node) + if node.children[0].value == '(': + if node.children[-1].value == ')': + if pytree_utils.NodeName(node.parent) == 'if_stmt': + _SetSplitPenalty(node.children[-1], STRONGLY_CONNECTED) + else: + if len(node.children) > 2: + _SetSplitPenalty(pytree_utils.FirstLeafNode(node.children[1]), EXPR) + _SetSplitPenalty(node.children[-1], ATOM) + elif node.children[0].value in '[{' and len(node.children) == 2: + # Keep empty containers together if we can. + _SetUnbreakable(node.children[-1]) + + def Visit_testlist_gexp(self, node): # pylint: disable=invalid-name + self.DefaultNodeVisit(node) + prev_was_comma = False + for child in node.children: + if isinstance(child, pytree.Leaf) and child.value == ',': + _SetUnbreakable(child) + prev_was_comma = True + else: + if prev_was_comma: + _SetSplitPenalty(pytree_utils.FirstLeafNode(child), 0) + prev_was_comma = False + + ############################################################################ + # Helper methods that set the annotations. + + def _SetUnbreakableOnChildren(self, node): + """Set an UNBREAKABLE penalty annotation on children of node.""" + for child in node.children: + self.Visit(child) + start = 2 if hasattr(node.children[0], 'is_pseudo') else 1 + for i in py3compat.range(start, len(node.children)): + _SetUnbreakable(node.children[i]) + + +def _SetUnbreakable(node): + """Set an UNBREAKABLE penalty annotation for the given node.""" + _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, UNBREAKABLE) + + +def _SetStronglyConnected(*nodes): + """Set a STRONGLY_CONNECTED penalty annotation for the given nodes.""" + for node in nodes: + _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, + STRONGLY_CONNECTED) + + +def _SetVeryStronglyConnected(*nodes): + """Set a VERY_STRONGLY_CONNECTED penalty annotation for the given nodes.""" + for node in nodes: + _RecAnnotate(node, pytree_utils.Annotation.SPLIT_PENALTY, + VERY_STRONGLY_CONNECTED) + + +def _SetExpressionPenalty(node, penalty): + """Set a penalty annotation on children nodes.""" + + def RecExpression(node, first_child_leaf): + if node is first_child_leaf: + return + + if isinstance(node, pytree.Leaf): + if node.value in {'(', 'for', 'if'}: + return + penalty_annotation = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) + if penalty_annotation < penalty: + _SetSplitPenalty(node, penalty) + else: + for child in node.children: + RecExpression(child, first_child_leaf) + + RecExpression(node, pytree_utils.FirstLeafNode(node)) + + +def _IncreasePenalty(node, amt): + """Increase a penalty annotation on children nodes.""" + + def RecExpression(node, first_child_leaf): + if node is first_child_leaf: + return + + if isinstance(node, pytree.Leaf): + if node.value in {'(', 'for', 'if'}: + return + penalty = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=0) + _SetSplitPenalty(node, penalty + amt) + else: + for child in node.children: + RecExpression(child, first_child_leaf) + + RecExpression(node, pytree_utils.FirstLeafNode(node)) + + +def _RecAnnotate(tree, annotate_name, annotate_value): + """Recursively set the given annotation on all leafs of the subtree. + + Takes care to only increase the penalty. If the node already has a higher + or equal penalty associated with it, this is a no-op. + + Args: + tree: subtree to annotate + annotate_name: name of the annotation to set + annotate_value: value of the annotation to set + """ + for child in tree.children: + _RecAnnotate(child, annotate_name, annotate_value) + if isinstance(tree, pytree.Leaf): + cur_annotate = pytree_utils.GetNodeAnnotation( + tree, annotate_name, default=0) + if cur_annotate < annotate_value: + pytree_utils.SetNodeAnnotation(tree, annotate_name, annotate_value) + + +def _StronglyConnectedCompOp(op): + if (len(op.children[1].children) == 2 and + pytree_utils.NodeName(op.children[1]) == 'comp_op' and + pytree_utils.FirstLeafNode(op.children[1]).value == 'not' and + pytree_utils.LastLeafNode(op.children[1]).value == 'in'): + return True + if (isinstance(op.children[1], pytree.Leaf) and + op.children[1].value in {'==', 'in'}): + return True + return False + + +def _DecrementSplitPenalty(node, amt): + penalty = pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SPLIT_PENALTY, default=amt) + penalty = penalty - amt if amt < penalty else 0 + _SetSplitPenalty(node, penalty) + + +def _SetSplitPenalty(node, penalty): + pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.SPLIT_PENALTY, + penalty) + + +def _AllowBuilderStyleCalls(node): + """Allow splitting before '.' if it's a builder style function call.""" + + def RecGetLeaves(node): + if isinstance(node, pytree.Leaf): + return [node] + children = [] + for child in node.children: + children += RecGetLeaves(child) + return children + + list_of_children = RecGetLeaves(node) + prev_child = None + for child in list_of_children: + if child.value == '.': + if prev_child.lineno != child.lineno: + _SetSplitPenalty(child, 0) + prev_child = child diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py new file mode 100644 index 0000000..6144246 --- /dev/null +++ b/yapf/yapflib/style.py @@ -0,0 +1,615 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python formatting style settings.""" + +import os +import re +import textwrap + +from yapf.yapflib import errors +from yapf.yapflib import py3compat + + +class StyleConfigError(errors.YapfError): + """Raised when there's a problem reading the style configuration.""" + pass + + +def Get(setting_name): + """Get a style setting.""" + return _style[setting_name] + + +def Help(): + """Return dict mapping style names to help strings.""" + return _STYLE_HELP + + +def SetGlobalStyle(style): + """Set a style dict.""" + global _style + global _GLOBAL_STYLE_FACTORY + factory = _GetStyleFactory(style) + if factory: + _GLOBAL_STYLE_FACTORY = factory + _style = style + + +_STYLE_HELP = dict( + ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=textwrap.dedent("""\ + Align closing bracket with visual indentation."""), + ALLOW_MULTILINE_LAMBDAS=textwrap.dedent("""\ + Allow lambdas to be formatted on more than one line."""), + ALLOW_MULTILINE_DICTIONARY_KEYS=textwrap.dedent("""\ + Allow dictionary keys to exist on multiple lines. For example: + + x = { + ('this is the first element of a tuple', + 'this is the second element of a tuple'): + value, + }"""), + ALLOW_SPLIT_BEFORE_DICT_VALUE=textwrap.dedent("""\ + Allow splits before the dictionary value."""), + BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=textwrap.dedent("""\ + Insert a blank line before a 'def' or 'class' immediately nested + within another 'def' or 'class'. For example: + + class Foo: + # <------ this blank line + def method(): + ..."""), + BLANK_LINE_BEFORE_CLASS_DOCSTRING=textwrap.dedent("""\ + Insert a blank line before a class-level docstring."""), + BLANK_LINE_BEFORE_MODULE_DOCSTRING=textwrap.dedent("""\ + Insert a blank line before a module docstring."""), + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=textwrap.dedent("""\ + Number of blank lines surrounding top-level function and class + definitions."""), + COALESCE_BRACKETS=textwrap.dedent("""\ + Do not split consecutive brackets. Only relevant when + dedent_closing_brackets is set. For example: + + call_func_that_takes_a_dict( + { + 'key1': 'value1', + 'key2': 'value2', + } + ) + + would reformat to: + + call_func_that_takes_a_dict({ + 'key1': 'value1', + 'key2': 'value2', + })"""), + COLUMN_LIMIT=textwrap.dedent("""\ + The column limit."""), + CONTINUATION_ALIGN_STYLE=textwrap.dedent("""\ + The style for continuation alignment. Possible values are: + + - SPACE: Use spaces for continuation alignment. This is default behavior. + - FIXED: Use fixed number (CONTINUATION_INDENT_WIDTH) of columns + (ie: CONTINUATION_INDENT_WIDTH/INDENT_WIDTH tabs) for continuation + alignment. + - LESS: Slightly left if cannot vertically align continuation lines with + indent characters. + - VALIGN-RIGHT: Vertically align continuation lines with indent + characters. Slightly right (one more indent character) if cannot + vertically align continuation lines with indent characters. + + For options FIXED, and VALIGN-RIGHT are only available when USE_TABS is + enabled."""), + CONTINUATION_INDENT_WIDTH=textwrap.dedent("""\ + Indent width used for line continuations."""), + DEDENT_CLOSING_BRACKETS=textwrap.dedent("""\ + Put closing brackets on a separate line, dedented, if the bracketed + expression can't fit in a single line. Applies to all kinds of brackets, + including function definitions and calls. For example: + + config = { + 'key1': 'value1', + 'key2': 'value2', + } # <--- this bracket is dedented and on a separate line + + time_series = self.remote_client.query_entity_counters( + entity='dev3246.region1', + key='dns.query_latency_tcp', + transform=Transformation.AVERAGE(window=timedelta(seconds=60)), + start_ts=now()-timedelta(days=3), + end_ts=now(), + ) # <--- this bracket is dedented and on a separate line"""), + DISABLE_ENDING_COMMA_HEURISTIC=textwrap.dedent("""\ + Disable the heuristic which places each list element on a separate line + if the list is comma-terminated."""), + EACH_DICT_ENTRY_ON_SEPARATE_LINE=textwrap.dedent("""\ + Place each dictionary entry onto its own line."""), + I18N_COMMENT=textwrap.dedent("""\ + The regex for an i18n comment. The presence of this comment stops + reformatting of that line, because the comments are required to be + next to the string they translate."""), + I18N_FUNCTION_CALL=textwrap.dedent("""\ + The i18n function call names. The presence of this function stops + reformattting on that line, because the string it has cannot be moved + away from the i18n comment."""), + INDENT_DICTIONARY_VALUE=textwrap.dedent("""\ + Indent the dictionary value if it cannot fit on the same line as the + dictionary key. For example: + + config = { + 'key1': + 'value1', + 'key2': value1 + + value2, + }"""), + INDENT_WIDTH=textwrap.dedent("""\ + The number of columns to use for indentation."""), + JOIN_MULTIPLE_LINES=textwrap.dedent("""\ + Join short lines into one line. E.g., single line 'if' statements."""), + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=textwrap.dedent("""\ + Do not include spaces around selected binary operators. For example: + + 1 + 2 * 3 - 4 / 5 + + will be formatted as follows when configured with *,/: + + 1 + 2*3 - 4/5 + + """), + SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=textwrap.dedent("""\ + Insert a space between the ending comma and closing bracket of a list, + etc."""), + SPACES_AROUND_POWER_OPERATOR=textwrap.dedent("""\ + Use spaces around the power operator."""), + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=textwrap.dedent("""\ + Use spaces around default or named assigns."""), + SPACES_BEFORE_COMMENT=textwrap.dedent("""\ + The number of spaces required before a trailing comment."""), + SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=textwrap.dedent("""\ + Split before arguments if the argument list is terminated by a + comma."""), + SPLIT_ALL_COMMA_SEPARATED_VALUES=textwrap.dedent("""\ + Split before arguments"""), + SPLIT_BEFORE_BITWISE_OPERATOR=textwrap.dedent("""\ + Set to True to prefer splitting before '&', '|' or '^' rather than + after."""), + SPLIT_BEFORE_CLOSING_BRACKET=textwrap.dedent("""\ + Split before the closing bracket if a list or dict literal doesn't fit on + a single line."""), + SPLIT_BEFORE_DICT_SET_GENERATOR=textwrap.dedent("""\ + Split before a dictionary or set generator (comp_for). For example, note + the split before the 'for': + + foo = { + variable: 'Hello world, have a nice day!' + for variable in bar if variable != 42 + }"""), + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=textwrap.dedent("""\ + Split after the opening paren which surrounds an expression if it doesn't + fit on a single line. + """), + SPLIT_BEFORE_FIRST_ARGUMENT=textwrap.dedent("""\ + If an argument / parameter list is going to be split, then split before + the first argument."""), + SPLIT_BEFORE_LOGICAL_OPERATOR=textwrap.dedent("""\ + Set to True to prefer splitting before 'and' or 'or' rather than + after."""), + SPLIT_BEFORE_NAMED_ASSIGNS=textwrap.dedent("""\ + Split named assignments onto individual lines."""), + SPLIT_COMPLEX_COMPREHENSION=textwrap.dedent("""\ + Set to True to split list comprehensions and generators that have + non-trivial expressions and multiple clauses before each of these + clauses. For example: + + result = [ + a_long_var + 100 for a_long_var in xrange(1000) + if a_long_var % 10] + + would reformat to something like: + + result = [ + a_long_var + 100 + for a_long_var in xrange(1000) + if a_long_var % 10] + """), + SPLIT_PENALTY_AFTER_OPENING_BRACKET=textwrap.dedent("""\ + The penalty for splitting right after the opening bracket."""), + SPLIT_PENALTY_AFTER_UNARY_OPERATOR=textwrap.dedent("""\ + The penalty for splitting the line after a unary operator."""), + SPLIT_PENALTY_BEFORE_IF_EXPR=textwrap.dedent("""\ + The penalty for splitting right before an if expression."""), + SPLIT_PENALTY_BITWISE_OPERATOR=textwrap.dedent("""\ + The penalty of splitting the line around the '&', '|', and '^' + operators."""), + SPLIT_PENALTY_COMPREHENSION=textwrap.dedent("""\ + The penalty for splitting a list comprehension or generator + expression."""), + SPLIT_PENALTY_EXCESS_CHARACTER=textwrap.dedent("""\ + The penalty for characters over the column limit."""), + SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=textwrap.dedent("""\ + The penalty incurred by adding a line split to the unwrapped line. The + more line splits added the higher the penalty."""), + SPLIT_PENALTY_IMPORT_NAMES=textwrap.dedent("""\ + The penalty of splitting a list of "import as" names. For example: + + from a_very_long_or_indented_module_name_yada_yad import (long_argument_1, + long_argument_2, + long_argument_3) + + would reformat to something like: + + from a_very_long_or_indented_module_name_yada_yad import ( + long_argument_1, long_argument_2, long_argument_3) + """), + SPLIT_PENALTY_LOGICAL_OPERATOR=textwrap.dedent("""\ + The penalty of splitting the line around the 'and' and 'or' + operators."""), + USE_TABS=textwrap.dedent("""\ + Use the Tab character for indentation."""), + # BASED_ON_STYLE='Which predefined style this style is based on', +) + + +def CreatePEP8Style(): + return dict( + ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True, + ALLOW_MULTILINE_LAMBDAS=False, + ALLOW_MULTILINE_DICTIONARY_KEYS=False, + ALLOW_SPLIT_BEFORE_DICT_VALUE=True, + BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=False, + BLANK_LINE_BEFORE_CLASS_DOCSTRING=False, + BLANK_LINE_BEFORE_MODULE_DOCSTRING=False, + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=2, + COALESCE_BRACKETS=False, + COLUMN_LIMIT=79, + CONTINUATION_ALIGN_STYLE='SPACE', + CONTINUATION_INDENT_WIDTH=4, + DEDENT_CLOSING_BRACKETS=False, + DISABLE_ENDING_COMMA_HEURISTIC=False, + EACH_DICT_ENTRY_ON_SEPARATE_LINE=True, + I18N_COMMENT='', + I18N_FUNCTION_CALL='', + INDENT_DICTIONARY_VALUE=False, + INDENT_WIDTH=4, + JOIN_MULTIPLE_LINES=True, + SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=True, + SPACES_AROUND_POWER_OPERATOR=False, + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=set(), + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=False, + SPACES_BEFORE_COMMENT=2, + SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=False, + SPLIT_ALL_COMMA_SEPARATED_VALUES=False, + SPLIT_BEFORE_BITWISE_OPERATOR=True, + SPLIT_BEFORE_CLOSING_BRACKET=True, + SPLIT_BEFORE_DICT_SET_GENERATOR=True, + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=False, + SPLIT_BEFORE_FIRST_ARGUMENT=False, + SPLIT_BEFORE_LOGICAL_OPERATOR=True, + SPLIT_BEFORE_NAMED_ASSIGNS=True, + SPLIT_COMPLEX_COMPREHENSION=False, + SPLIT_PENALTY_AFTER_OPENING_BRACKET=30, + SPLIT_PENALTY_AFTER_UNARY_OPERATOR=10000, + SPLIT_PENALTY_BEFORE_IF_EXPR=0, + SPLIT_PENALTY_BITWISE_OPERATOR=300, + SPLIT_PENALTY_COMPREHENSION=80, + SPLIT_PENALTY_EXCESS_CHARACTER=4500, + SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=30, + SPLIT_PENALTY_IMPORT_NAMES=0, + SPLIT_PENALTY_LOGICAL_OPERATOR=300, + USE_TABS=False, + ) + + +def CreateGoogleStyle(): + style = CreatePEP8Style() + style['ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'] = False + style['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF'] = True + style['COLUMN_LIMIT'] = 80 + style['INDENT_WIDTH'] = 4 + style['I18N_COMMENT'] = r'#\..*' + style['I18N_FUNCTION_CALL'] = ['N_', '_'] + style['SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET'] = False + style['SPLIT_BEFORE_BITWISE_OPERATOR'] = False + style['SPLIT_BEFORE_DICT_SET_GENERATOR'] = False + style['SPLIT_BEFORE_LOGICAL_OPERATOR'] = False + style['SPLIT_COMPLEX_COMPREHENSION'] = True + style['SPLIT_PENALTY_COMPREHENSION'] = 2100 + return style + + +def CreateChromiumStyle(): + style = CreateGoogleStyle() + style['ALLOW_MULTILINE_DICTIONARY_KEYS'] = True + style['INDENT_DICTIONARY_VALUE'] = True + style['INDENT_WIDTH'] = 2 + style['JOIN_MULTIPLE_LINES'] = False + style['SPLIT_BEFORE_BITWISE_OPERATOR'] = True + style['SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN'] = True + return style + + +def CreateFacebookStyle(): + style = CreatePEP8Style() + style['ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'] = False + style['COLUMN_LIMIT'] = 80 + style['DEDENT_CLOSING_BRACKETS'] = True + style['INDENT_DICTIONARY_VALUE'] = True + style['JOIN_MULTIPLE_LINES'] = False + style['SPACES_BEFORE_COMMENT'] = 2 + style['SPLIT_PENALTY_AFTER_OPENING_BRACKET'] = 0 + style['SPLIT_PENALTY_BEFORE_IF_EXPR'] = 30 + style['SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT'] = 30 + style['SPLIT_BEFORE_LOGICAL_OPERATOR'] = False + style['SPLIT_BEFORE_BITWISE_OPERATOR'] = False + return style + + +_STYLE_NAME_TO_FACTORY = dict( + pep8=CreatePEP8Style, + chromium=CreateChromiumStyle, + google=CreateGoogleStyle, + facebook=CreateFacebookStyle, +) + +_DEFAULT_STYLE_TO_FACTORY = [ + (CreateChromiumStyle(), CreateChromiumStyle), + (CreateFacebookStyle(), CreateFacebookStyle), + (CreateGoogleStyle(), CreateGoogleStyle), + (CreatePEP8Style(), CreatePEP8Style), +] + + +def _GetStyleFactory(style): + for def_style, factory in _DEFAULT_STYLE_TO_FACTORY: + if style == def_style: + return factory + return None + + +def _ContinuationAlignStyleStringConverter(s): + """Option value converter for a continuation align style string.""" + accepted_styles = ('SPACE', 'FIXED', 'VALIGN-RIGHT') + if s: + r = s.upper() + if r not in accepted_styles: + raise ValueError('unknown continuation align style: %r' % (s,)) + else: + r = accepted_styles[0] + return r + + +def _StringListConverter(s): + """Option value converter for a comma-separated list of strings.""" + return [part.strip() for part in s.split(',')] + + +def _StringSetConverter(s): + """Option value converter for a comma-separated set of strings.""" + return set(part.strip() for part in s.split(',')) + + +def _BoolConverter(s): + """Option value converter for a boolean.""" + return py3compat.CONFIGPARSER_BOOLEAN_STATES[s.lower()] + + +# Different style options need to have their values interpreted differently when +# read from the config file. This dict maps an option name to a "converter" +# function that accepts the string read for the option's value from the file and +# returns it wrapper in actual Python type that's going to be meaningful to +# yapf. +# +# Note: this dict has to map all the supported style options. +_STYLE_OPTION_VALUE_CONVERTER = dict( + ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=_BoolConverter, + ALLOW_MULTILINE_LAMBDAS=_BoolConverter, + ALLOW_MULTILINE_DICTIONARY_KEYS=_BoolConverter, + ALLOW_SPLIT_BEFORE_DICT_VALUE=_BoolConverter, + BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF=_BoolConverter, + BLANK_LINE_BEFORE_CLASS_DOCSTRING=_BoolConverter, + BLANK_LINE_BEFORE_MODULE_DOCSTRING=_BoolConverter, + BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION=int, + COALESCE_BRACKETS=_BoolConverter, + COLUMN_LIMIT=int, + CONTINUATION_ALIGN_STYLE=_ContinuationAlignStyleStringConverter, + CONTINUATION_INDENT_WIDTH=int, + DEDENT_CLOSING_BRACKETS=_BoolConverter, + DISABLE_ENDING_COMMA_HEURISTIC=_BoolConverter, + EACH_DICT_ENTRY_ON_SEPARATE_LINE=_BoolConverter, + I18N_COMMENT=str, + I18N_FUNCTION_CALL=_StringListConverter, + INDENT_DICTIONARY_VALUE=_BoolConverter, + INDENT_WIDTH=int, + JOIN_MULTIPLE_LINES=_BoolConverter, + NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS=_StringSetConverter, + SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET=_BoolConverter, + SPACES_AROUND_POWER_OPERATOR=_BoolConverter, + SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN=_BoolConverter, + SPACES_BEFORE_COMMENT=int, + SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED=_BoolConverter, + SPLIT_ALL_COMMA_SEPARATED_VALUES=_BoolConverter, + SPLIT_BEFORE_BITWISE_OPERATOR=_BoolConverter, + SPLIT_BEFORE_CLOSING_BRACKET=_BoolConverter, + SPLIT_BEFORE_DICT_SET_GENERATOR=_BoolConverter, + SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN=_BoolConverter, + SPLIT_BEFORE_FIRST_ARGUMENT=_BoolConverter, + SPLIT_BEFORE_LOGICAL_OPERATOR=_BoolConverter, + SPLIT_BEFORE_NAMED_ASSIGNS=_BoolConverter, + SPLIT_COMPLEX_COMPREHENSION=_BoolConverter, + SPLIT_PENALTY_AFTER_OPENING_BRACKET=int, + SPLIT_PENALTY_AFTER_UNARY_OPERATOR=int, + SPLIT_PENALTY_BEFORE_IF_EXPR=int, + SPLIT_PENALTY_BITWISE_OPERATOR=int, + SPLIT_PENALTY_COMPREHENSION=int, + SPLIT_PENALTY_EXCESS_CHARACTER=int, + SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT=int, + SPLIT_PENALTY_IMPORT_NAMES=int, + SPLIT_PENALTY_LOGICAL_OPERATOR=int, + USE_TABS=_BoolConverter, +) + + +def CreateStyleFromConfig(style_config): + """Create a style dict from the given config. + + Arguments: + style_config: either a style name or a file name. The file is expected to + contain settings. It can have a special BASED_ON_STYLE setting naming the + style which it derives from. If no such setting is found, it derives from + the default style. When style_config is None, the _GLOBAL_STYLE_FACTORY + config is created. + + Returns: + A style dict. + + Raises: + StyleConfigError: if an unknown style option was encountered. + """ + + def GlobalStyles(): + for style, _ in _DEFAULT_STYLE_TO_FACTORY: + yield style + + def_style = False + if style_config is None: + for style in GlobalStyles(): + if _style == style: + def_style = True + break + if not def_style: + return _style + return _GLOBAL_STYLE_FACTORY() + if isinstance(style_config, dict): + config = _CreateConfigParserFromConfigDict(style_config) + elif isinstance(style_config, py3compat.basestring): + style_factory = _STYLE_NAME_TO_FACTORY.get(style_config.lower()) + if style_factory is not None: + return style_factory() + if style_config.startswith('{'): + # Most likely a style specification from the command line. + config = _CreateConfigParserFromConfigString(style_config) + else: + # Unknown config name: assume it's a file name then. + config = _CreateConfigParserFromConfigFile(style_config) + return _CreateStyleFromConfigParser(config) + + +def _CreateConfigParserFromConfigDict(config_dict): + config = py3compat.ConfigParser() + config.add_section('style') + for key, value in config_dict.items(): + config.set('style', key, str(value)) + return config + + +def _CreateConfigParserFromConfigString(config_string): + """Given a config string from the command line, return a config parser.""" + if config_string[0] != '{' or config_string[-1] != '}': + raise StyleConfigError( + "Invalid style dict syntax: '{}'.".format(config_string)) + config = py3compat.ConfigParser() + config.add_section('style') + for key, value in re.findall(r'([a-zA-Z0-9_]+)\s*[:=]\s*([a-zA-Z0-9_]+)', + config_string): + config.set('style', key, value) + return config + + +def _CreateConfigParserFromConfigFile(config_filename): + """Read the file and return a ConfigParser object.""" + if not os.path.exists(config_filename): + # Provide a more meaningful error here. + raise StyleConfigError( + '"{0}" is not a valid style or file path'.format(config_filename)) + with open(config_filename) as style_file: + config = py3compat.ConfigParser() + config.read_file(style_file) + if config_filename.endswith(SETUP_CONFIG): + if not config.has_section('yapf'): + raise StyleConfigError( + 'Unable to find section [yapf] in {0}'.format(config_filename)) + elif config_filename.endswith(LOCAL_STYLE): + if not config.has_section('style'): + raise StyleConfigError( + 'Unable to find section [style] in {0}'.format(config_filename)) + else: + if not config.has_section('style'): + raise StyleConfigError( + 'Unable to find section [style] in {0}'.format(config_filename)) + return config + + +def _CreateStyleFromConfigParser(config): + """Create a style dict from a configuration file. + + Arguments: + config: a ConfigParser object. + + Returns: + A style dict. + + Raises: + StyleConfigError: if an unknown style option was encountered. + """ + # Initialize the base style. + section = 'yapf' if config.has_section('yapf') else 'style' + if config.has_option('style', 'based_on_style'): + based_on = config.get('style', 'based_on_style').lower() + base_style = _STYLE_NAME_TO_FACTORY[based_on]() + elif config.has_option('yapf', 'based_on_style'): + based_on = config.get('yapf', 'based_on_style').lower() + base_style = _STYLE_NAME_TO_FACTORY[based_on]() + else: + base_style = _GLOBAL_STYLE_FACTORY() + + # Read all options specified in the file and update the style. + for option, value in config.items(section): + if option.lower() == 'based_on_style': + # Now skip this one - we've already handled it and it's not one of the + # recognized style options. + continue + option = option.upper() + if option not in _STYLE_OPTION_VALUE_CONVERTER: + raise StyleConfigError('Unknown style option "{0}"'.format(option)) + try: + base_style[option] = _STYLE_OPTION_VALUE_CONVERTER[option](value) + except ValueError: + raise StyleConfigError("'{}' is not a valid setting for {}.".format( + value, option)) + return base_style + + +# The default style - used if yapf is not invoked without specifically +# requesting a formatting style. +DEFAULT_STYLE = 'pep8' +DEFAULT_STYLE_FACTORY = CreatePEP8Style +_GLOBAL_STYLE_FACTORY = CreatePEP8Style + +# The name of the file to use for global style definition. +GLOBAL_STYLE = ( + os.path.join( + os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), 'yapf', + 'style')) + +# The name of the file to use for directory-local style definition. +LOCAL_STYLE = '.style.yapf' + +# Alternative place for directory-local style definition. Style should be +# specified in the '[yapf]' section. +SETUP_CONFIG = 'setup.cfg' + +# TODO(eliben): For now we're preserving the global presence of a style dict. +# Refactor this so that the style is passed around through yapf rather than +# being global. +_style = None +SetGlobalStyle(_GLOBAL_STYLE_FACTORY()) diff --git a/yapf/yapflib/subtype_assigner.py b/yapf/yapflib/subtype_assigner.py new file mode 100644 index 0000000..8bf3d8d --- /dev/null +++ b/yapf/yapflib/subtype_assigner.py @@ -0,0 +1,428 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Subtype assigner for lib2to3 trees. + +This module assigns extra type information to the lib2to3 trees. This +information is more specific than whether something is an operator or an +identifier. For instance, it can specify if a node in the tree is part of a +subscript. + + AssignSubtypes(): the main function exported by this module. + +Annotations: + subtype: The subtype of a pytree token. See 'format_token' module for a list + of subtypes. +""" + +from lib2to3 import pytree +from lib2to3.pgen2 import token +from lib2to3.pygram import python_symbols as syms + +from yapf.yapflib import format_token +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import style + + +def AssignSubtypes(tree): + """Run the subtype assigner visitor over the tree, modifying it in place. + + Arguments: + tree: the top-level pytree node to annotate with subtypes. + """ + subtype_assigner = _SubtypeAssigner() + subtype_assigner.Visit(tree) + + +# Map tokens in argument lists to their respective subtype. +_ARGLIST_TOKEN_TO_SUBTYPE = { + '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, + ':': format_token.Subtype.TYPED_NAME, + '*': format_token.Subtype.VARARGS_STAR, + '**': format_token.Subtype.KWARGS_STAR_STAR, +} + + +class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): + """_SubtypeAssigner - see file-level docstring for detailed description. + + The subtype is added as an annotation to the pytree token. + """ + + def Visit_dictsetmaker(self, node): # pylint: disable=invalid-name + # dictsetmaker ::= (test ':' test (comp_for | + # (',' test ':' test)* [','])) | + # (test (comp_for | (',' test)* [','])) + for child in node.children: + self.Visit(child) + + comp_for = False + dict_maker = False + + for child in node.children: + if pytree_utils.NodeName(child) == 'comp_for': + comp_for = True + _AppendFirstLeafTokenSubtype(child, + format_token.Subtype.DICT_SET_GENERATOR) + elif pytree_utils.NodeName(child) in ('COLON', 'DOUBLESTAR'): + dict_maker = True + + if not comp_for and dict_maker: + last_was_colon = False + unpacking = False + for child in node.children: + if pytree_utils.NodeName(child) == 'DOUBLESTAR': + _AppendFirstLeafTokenSubtype(child, + format_token.Subtype.KWARGS_STAR_STAR) + if last_was_colon: + if style.Get('INDENT_DICTIONARY_VALUE'): + _InsertPseudoParentheses(child) + else: + _AppendFirstLeafTokenSubtype(child, + format_token.Subtype.DICTIONARY_VALUE) + elif (isinstance(child, pytree.Node) or + (not child.value.startswith('#') and child.value not in '{:,')): + # Mark the first leaf of a key entry as a DICTIONARY_KEY. We + # normally want to split before them if the dictionary cannot exist + # on a single line. + if not unpacking or pytree_utils.FirstLeafNode(child).value == '**': + _AppendFirstLeafTokenSubtype(child, + format_token.Subtype.DICTIONARY_KEY) + _AppendSubtypeRec(child, format_token.Subtype.DICTIONARY_KEY_PART) + last_was_colon = pytree_utils.NodeName(child) == 'COLON' + if pytree_utils.NodeName(child) == 'DOUBLESTAR': + unpacking = True + elif last_was_colon: + unpacking = False + + def Visit_expr_stmt(self, node): # pylint: disable=invalid-name + # expr_stmt ::= testlist_star_expr (augassign (yield_expr|testlist) + # | ('=' (yield_expr|testlist_star_expr))*) + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '=': + _AppendTokenSubtype(child, format_token.Subtype.ASSIGN_OPERATOR) + + def Visit_or_test(self, node): # pylint: disable=invalid-name + # or_test ::= and_test ('or' and_test)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'or': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_and_test(self, node): # pylint: disable=invalid-name + # and_test ::= not_test ('and' not_test)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'and': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_not_test(self, node): # pylint: disable=invalid-name + # not_test ::= 'not' not_test | comparison + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == 'not': + _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) + + def Visit_comparison(self, node): # pylint: disable=invalid-name + # comparison ::= expr (comp_op expr)* + # comp_op ::= '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not in'|'is'|'is not' + for child in node.children: + self.Visit(child) + if (isinstance(child, pytree.Leaf) and + child.value in {'<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is'}): + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + elif pytree_utils.NodeName(child) == 'comp_op': + for grandchild in child.children: + _AppendTokenSubtype(grandchild, format_token.Subtype.BINARY_OPERATOR) + + def Visit_star_expr(self, node): # pylint: disable=invalid-name + # star_expr ::= '*' expr + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '*': + _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) + _AppendTokenSubtype(child, format_token.Subtype.VARARGS_STAR) + + def Visit_expr(self, node): # pylint: disable=invalid-name + # expr ::= xor_expr ('|' xor_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '|': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_xor_expr(self, node): # pylint: disable=invalid-name + # xor_expr ::= and_expr ('^' and_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '^': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_and_expr(self, node): # pylint: disable=invalid-name + # and_expr ::= shift_expr ('&' shift_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '&': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_shift_expr(self, node): # pylint: disable=invalid-name + # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in {'<<', '>>'}: + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_arith_expr(self, node): # pylint: disable=invalid-name + # arith_expr ::= term (('+'|'-') term)* + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in '+-': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_term(self, node): # pylint: disable=invalid-name + # term ::= factor (('*'|'/'|'%'|'//') factor)* + for child in node.children: + self.Visit(child) + if (isinstance(child, pytree.Leaf) and + child.value in {'*', '/', '%', '//'}): + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_factor(self, node): # pylint: disable=invalid-name + # factor ::= ('+'|'-'|'~') factor | power + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in '+-~': + _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) + + def Visit_power(self, node): # pylint: disable=invalid-name + # power ::= atom trailer* ['**' factor] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '**': + _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + + def Visit_trailer(self, node): # pylint: disable=invalid-name + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value in '[]': + _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_BRACKET) + + def Visit_subscript(self, node): # pylint: disable=invalid-name + # subscript ::= test | [test] ':' [test] [sliceop] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == ':': + _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) + + def Visit_sliceop(self, node): # pylint: disable=invalid-name + # sliceop ::= ':' [test] + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == ':': + _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) + + def Visit_argument(self, node): # pylint: disable=invalid-name + # argument ::= + # test [comp_for] | test '=' test + self._ProcessArgLists(node) + + def Visit_arglist(self, node): # pylint: disable=invalid-name + # arglist ::= + # (argument ',')* (argument [','] + # | '*' test (',' argument)* [',' '**' test] + # | '**' test) + self._ProcessArgLists(node) + _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + + def Visit_tname(self, node): # pylint: disable=invalid-name + self._ProcessArgLists(node) + _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + + def Visit_decorator(self, node): # pylint: disable=invalid-name + # decorator ::= + # '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + for child in node.children: + if isinstance(child, pytree.Leaf) and child.value == '@': + _AppendTokenSubtype(child, subtype=format_token.Subtype.DECORATOR) + self.Visit(child) + + def Visit_funcdef(self, node): # pylint: disable=invalid-name + # funcdef ::= + # 'def' NAME parameters ['->' test] ':' suite + for child in node.children: + if pytree_utils.NodeName(child) == 'NAME' and child.value != 'def': + _AppendTokenSubtype(child, format_token.Subtype.FUNC_DEF) + break + for child in node.children: + self.Visit(child) + + def Visit_typedargslist(self, node): # pylint: disable=invalid-name + # typedargslist ::= + # ((tfpdef ['=' test] ',')* + # ('*' [tname] (',' tname ['=' test])* [',' '**' tname] + # | '**' tname) + # | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + self._ProcessArgLists(node) + _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + tname = False + for child in node.children: + if pytree_utils.NodeName(child) == 'tname': + tname = True + _SetArgListSubtype(child, format_token.Subtype.TYPED_NAME, + format_token.Subtype.TYPED_NAME_ARG_LIST) + if not isinstance(child, pytree.Leaf): + continue + if child.value == ',': + tname = False + elif child.value == '=' and tname: + _AppendTokenSubtype(child, subtype=format_token.Subtype.TYPED_NAME) + tname = False + + def Visit_varargslist(self, node): # pylint: disable=invalid-name + # varargslist ::= + # ((vfpdef ['=' test] ',')* + # ('*' [vname] (',' vname ['=' test])* [',' '**' vname] + # | '**' vname) + # | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + self._ProcessArgLists(node) + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf) and child.value == '=': + _AppendTokenSubtype(child, format_token.Subtype.VARARGS_LIST) + + def Visit_comp_for(self, node): # pylint: disable=invalid-name + # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] + _AppendSubtypeRec(node, format_token.Subtype.COMP_FOR) + # Mark the previous node as COMP_EXPR unless this is a nested comprehension + # as these will have the outer comprehension as their previous node. + attr = pytree_utils.GetNodeAnnotation(node.parent, + pytree_utils.Annotation.SUBTYPE) + if not attr or format_token.Subtype.COMP_FOR not in attr: + _AppendSubtypeRec(node.parent.children[0], format_token.Subtype.COMP_EXPR) + self.DefaultNodeVisit(node) + + def Visit_comp_if(self, node): # pylint: disable=invalid-name + # comp_if ::= 'if' old_test [comp_iter] + _AppendSubtypeRec(node, format_token.Subtype.COMP_IF) + self.DefaultNodeVisit(node) + + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + self.Visit(child) + if isinstance(child, pytree.Leaf): + _AppendTokenSubtype( + child, + subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, + format_token.Subtype.NONE)) + + +def _SetArgListSubtype(node, node_subtype, list_subtype): + """Set named assign subtype on elements in a arg list.""" + + def HasSubtype(node): + """Return True if the arg list has a named assign subtype.""" + if isinstance(node, pytree.Leaf): + if node_subtype in pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SUBTYPE, set()): + return True + return False + has_subtype = False + for child in node.children: + if pytree_utils.NodeName(child) != 'arglist': + has_subtype |= HasSubtype(child) + return has_subtype + + if HasSubtype(node): + for child in node.children: + if pytree_utils.NodeName(child) != 'COMMA': + _AppendFirstLeafTokenSubtype(child, list_subtype) + + +def _AppendTokenSubtype(node, subtype): + """Append the token's subtype only if it's not already set.""" + pytree_utils.AppendNodeAnnotation(node, pytree_utils.Annotation.SUBTYPE, + subtype) + + +def _AppendFirstLeafTokenSubtype(node, subtype): + """Append the first leaf token's subtypes.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + _AppendFirstLeafTokenSubtype(node.children[0], subtype) + + +def _AppendSubtypeRec(node, subtype, force=True): + """Append the leafs in the node to the given subtype.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + for child in node.children: + _AppendSubtypeRec(child, subtype, force=force) + + +def _InsertPseudoParentheses(node): + """Insert pseudo parentheses so that dicts can be formatted correctly.""" + comment_node = None + if isinstance(node, pytree.Node): + if node.children[-1].type == token.COMMENT: + comment_node = node.children[-1].clone() + node.children[-1].remove() + + first = pytree_utils.FirstLeafNode(node) + last = pytree_utils.LastLeafNode(node) + + if first == last and first.type == token.COMMENT: + # A comment was inserted before the value, which is a pytree.Leaf. + # Encompass the dictionary's value into an ATOM node. + last = first.next_sibling + new_node = pytree.Node(syms.atom, [first.clone(), last.clone()]) + node.replace(new_node) + node = new_node + last.remove() + + first = pytree_utils.FirstLeafNode(node) + last = pytree_utils.LastLeafNode(node) + + lparen = pytree.Leaf( + token.LPAR, u'(', context=('', (first.get_lineno(), first.column - 1))) + last_lineno = last.get_lineno() + if last.type == token.STRING and '\n' in last.value: + last_lineno += last.value.count('\n') + + if last.type == token.STRING and '\n' in last.value: + last_column = len(last.value.split('\n')[-1]) + 1 + else: + last_column = last.column + len(last.value) + 1 + rparen = pytree.Leaf( + token.RPAR, u')', context=('', (last_lineno, last_column))) + + lparen.is_pseudo = True + rparen.is_pseudo = True + + if isinstance(node, pytree.Node): + node.insert_child(0, lparen) + node.append_child(rparen) + if comment_node: + node.append_child(comment_node) + _AppendFirstLeafTokenSubtype(node, format_token.Subtype.DICTIONARY_VALUE) + else: + clone = node.clone() + new_node = pytree.Node(syms.atom, [lparen, clone, rparen]) + node.replace(new_node) + _AppendFirstLeafTokenSubtype(clone, format_token.Subtype.DICTIONARY_VALUE) diff --git a/yapf/yapflib/unwrapped_line.py b/yapf/yapflib/unwrapped_line.py new file mode 100644 index 0000000..92b986e --- /dev/null +++ b/yapf/yapflib/unwrapped_line.py @@ -0,0 +1,540 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""UnwrappedLine primitive for formatting. + +An unwrapped line is the containing data structure produced by the parser. It +collects all nodes (stored in FormatToken objects) that could appear on a +single line if there were no line length restrictions. It's then used by the +parser to perform the wrapping required to comply with the style guide. +""" + +from yapf.yapflib import format_token +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils +from yapf.yapflib import split_penalty +from yapf.yapflib import style + + +class UnwrappedLine(object): + """Represents a single unwrapped line in the output. + + Attributes: + depth: indentation depth of this line. This is just a numeric value used to + distinguish lines that are more deeply nested than others. It is not the + actual amount of spaces, which is style-dependent. + """ + + def __init__(self, depth, tokens=None): + """Constructor. + + Creates a new unwrapped line with the given depth an initial list of tokens. + Constructs the doubly-linked lists for format tokens using their built-in + next_token and previous_token attributes. + + Arguments: + depth: indentation depth of this line + tokens: initial list of tokens + """ + self.depth = depth + self._tokens = tokens or [] + self.disable = False + + if self._tokens: + # Set up a doubly linked list. + for index, tok in enumerate(self._tokens[1:]): + # Note, 'index' is the index to the previous token. + tok.previous_token = self._tokens[index] + self._tokens[index].next_token = tok + + def CalculateFormattingInformation(self): + """Calculate the split penalty and total length for the tokens.""" + # Say that the first token in the line should have a space before it. This + # means only that if this unwrapped line is joined with a predecessor line, + # then there will be a space between them. + self.first.spaces_required_before = 1 + self.first.total_length = len(self.first.value) + + prev_token = self.first + prev_length = self.first.total_length + for token in self._tokens[1:]: + if (token.spaces_required_before == 0 and + _SpaceRequiredBetween(prev_token, token)): + token.spaces_required_before = 1 + + tok_len = len(token.value) if not token.is_pseudo_paren else 0 + token.total_length = prev_length + tok_len + token.spaces_required_before + + # The split penalty has to be computed before {must|can}_break_before, + # because these may use it for their decision. + token.split_penalty += _SplitPenalty(prev_token, token) + token.must_break_before = _MustBreakBefore(prev_token, token) + token.can_break_before = ( + token.must_break_before or _CanBreakBefore(prev_token, token)) + + prev_length = token.total_length + prev_token = token + + def Split(self): + """Split the line at semicolons.""" + if not self.has_semicolon or self.disable: + return [self] + + uwlines = [] + uwline = UnwrappedLine(self.depth) + for tok in self._tokens: + if tok.value == ';': + uwlines.append(uwline) + uwline = UnwrappedLine(self.depth) + else: + uwline.AppendToken(tok) + + if uwline.tokens: + uwlines.append(uwline) + + for uwline in uwlines: + pytree_utils.SetNodeAnnotation(uwline.first.node, + pytree_utils.Annotation.MUST_SPLIT, True) + uwline.first.previous_token = None + uwline.last.next_token = None + + return uwlines + + ############################################################################ + # Token Access and Manipulation Methods # + ############################################################################ + + def AppendToken(self, token): + """Append a new FormatToken to the tokens contained in this line.""" + if self._tokens: + token.previous_token = self.last + self.last.next_token = token + self._tokens.append(token) + + def AppendNode(self, node): + """Convenience method to append a pytree node directly. + + Wraps the node with a FormatToken. + + Arguments: + node: the node to append + """ + self.AppendToken(format_token.FormatToken(node)) + + @property + def first(self): + """Returns the first non-whitespace token.""" + return self._tokens[0] + + @property + def last(self): + """Returns the last non-whitespace token.""" + return self._tokens[-1] + + ############################################################################ + # Token -> String Methods # + ############################################################################ + + def AsCode(self, indent_per_depth=2): + """Return a "code" representation of this line. + + The code representation shows how the line would be printed out as code. + + TODO(eliben): for now this is rudimentary for debugging - once we add + formatting capabilities, this method will have other uses (not all tokens + have spaces around them, for example). + + Arguments: + indent_per_depth: how much spaces to indend per depth level. + + Returns: + A string representing the line as code. + """ + indent = ' ' * indent_per_depth * self.depth + tokens_str = ' '.join(tok.value for tok in self._tokens) + return indent + tokens_str + + def __str__(self): # pragma: no cover + return self.AsCode() + + def __repr__(self): # pragma: no cover + tokens_repr = ','.join( + ['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) + return 'UnwrappedLine(depth={0}, tokens=[{1}])'.format( + self.depth, tokens_repr) + + ############################################################################ + # Properties # + ############################################################################ + + @property + def tokens(self): + """Access the tokens contained within this line. + + The caller must not modify the tokens list returned by this method. + + Returns: + List of tokens in this line. + """ + return self._tokens + + @property + def lineno(self): + """Return the line number of this unwrapped line. + + Returns: + The line number of the first token in this unwrapped line. + """ + return self.first.lineno + + @property + def is_comment(self): + return self.first.is_comment + + @property + def has_semicolon(self): + return any(tok.value == ';' for tok in self._tokens) + + +def _IsIdNumberStringToken(tok): + return tok.is_keyword or tok.is_name or tok.is_number or tok.is_string + + +def _IsUnaryOperator(tok): + return format_token.Subtype.UNARY_OPERATOR in tok.subtypes + + +def _SpaceRequiredBetween(left, right): + """Return True if a space is required between the left and right token.""" + lval = left.value + rval = right.value + if (left.is_pseudo_paren and _IsIdNumberStringToken(right) and + left.previous_token and _IsIdNumberStringToken(left.previous_token)): + # Space between keyword... tokens and pseudo parens. + return True + if left.is_pseudo_paren or right.is_pseudo_paren: + # There should be a space after the ':' in a dictionary. + if left.OpensScope(): + return True + # The closing pseudo-paren shouldn't affect spacing. + return False + if left.is_continuation or right.is_continuation: + # The continuation node's value has all of the spaces it needs. + return False + if right.name in pytree_utils.NONSEMANTIC_TOKENS: + # No space before a non-semantic token. + return False + if _IsIdNumberStringToken(left) and _IsIdNumberStringToken(right): + # Spaces between keyword, string, number, and identifier tokens. + return True + if lval == ',' and rval == ':': + # We do want a space between a comma and colon. + return True + if rval in ':,': + # Otherwise, we never want a space before a colon or comma. + return False + if lval == ',' and rval in ']})': + # Add a space between ending ',' and closing bracket if requested. + return style.Get('SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET') + if lval == ',': + # We want a space after a comma. + return True + if lval == 'from' and rval == '.': + # Space before the '.' in an import statement. + return True + if lval == '.' and rval == 'import': + # Space after the '.' in an import statement. + return True + if (lval == '=' and rval == '.' and + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN not in left.subtypes): + # Space between equal and '.' as in "X = ...". + return True + if ((right.is_keyword or right.is_name) and + (left.is_keyword or left.is_name)): + # Don't merge two keywords/identifiers. + return True + if (format_token.Subtype.SUBSCRIPT_COLON in left.subtypes or + format_token.Subtype.SUBSCRIPT_COLON in right.subtypes): + # A subscript shouldn't have spaces separating its colons. + return False + if (format_token.Subtype.TYPED_NAME in left.subtypes or + format_token.Subtype.TYPED_NAME in right.subtypes): + # A typed argument should have a space after the colon. + return True + if left.is_string: + if (rval == '=' and + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in right.subtypes + ): + # If there is a type hint, then we don't want to add a space between the + # equal sign and the hint. + return False + if rval not in '[)]}.': + # A string followed by something other than a subscript, closing bracket, + # or dot should have a space after it. + return True + if left.is_binary_op and lval != '**' and _IsUnaryOperator(right): + # Space between the binary operator and the unary operator. + return True + if left.is_keyword and _IsUnaryOperator(right): + # Handle things like "not -3 < x". + return True + if _IsUnaryOperator(left) and _IsUnaryOperator(right): + # No space between two unary operators. + return False + if left.is_binary_op or right.is_binary_op: + if lval == '**' or rval == '**': + # Space around the "power" operator. + return style.Get('SPACES_AROUND_POWER_OPERATOR') + # Enforce spaces around binary operators except the blacklisted ones. + blacklist = style.Get('NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS') + return lval not in blacklist and rval not in blacklist + if (_IsUnaryOperator(left) and lval != 'not' and + (right.is_name or right.is_number or rval == '(')): + # The previous token was a unary op. No space is desired between it and + # the current token. + return False + if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in left.subtypes or + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in right.subtypes): + # A named argument or default parameter shouldn't have spaces around it. + return style.Get('SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN') + if (format_token.Subtype.VARARGS_LIST in left.subtypes or + format_token.Subtype.VARARGS_LIST in right.subtypes): + return False + if (format_token.Subtype.VARARGS_STAR in left.subtypes or + format_token.Subtype.KWARGS_STAR_STAR in left.subtypes): + # Don't add a space after a vararg's star or a keyword's star-star. + return False + if lval == '@' and format_token.Subtype.DECORATOR in left.subtypes: + # Decorators shouldn't be separated from the 'at' sign. + return False + if left.is_keyword and rval == '.' or lval == '.' and right.is_keyword: + # Add space between keywords and dots. + return lval != 'None' + if lval == '.' or rval == '.': + # Don't place spaces between dots. + return False + if ((lval == '(' and rval == ')') or (lval == '[' and rval == ']') or + (lval == '{' and rval == '}')): + # Empty objects shouldn't be separated by spaces. + return False + if (lval in pytree_utils.OPENING_BRACKETS and + rval in pytree_utils.OPENING_BRACKETS): + # Nested objects' opening brackets shouldn't be separated. + return False + if (lval in pytree_utils.CLOSING_BRACKETS and + rval in pytree_utils.CLOSING_BRACKETS): + # Nested objects' closing brackets shouldn't be separated. + return False + if lval in pytree_utils.CLOSING_BRACKETS and rval in '([': + # A call, set, dictionary, or subscript that has a call or subscript after + # it shouldn't have a space between them. + return False + if lval in pytree_utils.OPENING_BRACKETS and _IsIdNumberStringToken(right): + # Don't separate the opening bracket from the first item. + return False + if left.is_name and rval in '([': + # Don't separate a call or array access from the name. + return False + if rval in pytree_utils.CLOSING_BRACKETS: + # Don't separate the closing bracket from the last item. + # FIXME(morbo): This might be too permissive. + return False + if lval == 'print' and rval == '(': + # Special support for the 'print' function. + return False + if lval in pytree_utils.OPENING_BRACKETS and _IsUnaryOperator(right): + # Don't separate a unary operator from the opening bracket. + return False + if (lval in pytree_utils.OPENING_BRACKETS and + (format_token.Subtype.VARARGS_STAR in right.subtypes or + format_token.Subtype.KWARGS_STAR_STAR in right.subtypes)): + # Don't separate a '*' or '**' from the opening bracket. + return False + if rval == ';': + # Avoid spaces before a semicolon. (Why is there a semicolon?!) + return False + if lval == '(' and rval == 'await': + # Special support for the 'await' keyword. Don't separate the 'await' + # keyword from an opening paren. + return False + return True + + +def _MustBreakBefore(prev_token, cur_token): + """Return True if a line break is required before the current token.""" + if prev_token.is_comment or (prev_token.previous_token and + prev_token.is_pseudo_paren and + prev_token.previous_token.is_comment): + # Must break if the previous token was a comment. + return True + if (cur_token.is_string and prev_token.is_string and + IsSurroundedByBrackets(cur_token)): + # We want consecutive strings to be on separate lines. This is a + # reasonable assumption, because otherwise they should have written them + # all on the same line, or with a '+'. + return True + return pytree_utils.GetNodeAnnotation( + cur_token.node, pytree_utils.Annotation.MUST_SPLIT, default=False) + + +def _CanBreakBefore(prev_token, cur_token): + """Return True if a line break may occur before the current token.""" + pval = prev_token.value + cval = cur_token.value + if py3compat.PY3: + if pval == 'yield' and cval == 'from': + # Don't break before a yield argument. + return False + if pval in {'async', 'await'} and cval in {'def', 'with', 'for'}: + # Don't break after sync keywords. + return False + if cur_token.split_penalty >= split_penalty.UNBREAKABLE: + return False + if pval == '@': + # Don't break right after the beginning of a decorator. + return False + if cval == ':': + # Don't break before the start of a block of code. + return False + if cval == ',': + # Don't break before a comma. + return False + if prev_token.is_name and cval == '(': + # Don't break in the middle of a function definition or call. + return False + if prev_token.is_name and cval == '[': + # Don't break in the middle of an array dereference. + return False + if prev_token.is_name and cval == '.': + # Don't break before the '.' in a dotted name. + return False + if cur_token.is_comment and prev_token.lineno == cur_token.lineno: + # Don't break a comment at the end of the line. + return False + if format_token.Subtype.UNARY_OPERATOR in prev_token.subtypes: + # Don't break after a unary token. + return False + return True + + +def IsSurroundedByBrackets(tok): + """Return True if the token is surrounded by brackets.""" + paren_count = 0 + brace_count = 0 + sq_bracket_count = 0 + previous_token = tok.previous_token + while previous_token: + if previous_token.value == ')': + paren_count -= 1 + elif previous_token.value == '}': + brace_count -= 1 + elif previous_token.value == ']': + sq_bracket_count -= 1 + + if previous_token.value == '(': + if paren_count == 0: + return previous_token + paren_count += 1 + elif previous_token.value == '{': + if brace_count == 0: + return previous_token + brace_count += 1 + elif previous_token.value == '[': + if sq_bracket_count == 0: + return previous_token + sq_bracket_count += 1 + + previous_token = previous_token.previous_token + return None + + +_LOGICAL_OPERATORS = frozenset({'and', 'or'}) +_BITWISE_OPERATORS = frozenset({'&', '|', '^'}) +_TERM_OPERATORS = frozenset({'*', '/', '%', '//'}) + + +def _SplitPenalty(prev_token, cur_token): + """Return the penalty for breaking the line before the current token.""" + pval = prev_token.value + cval = cur_token.value + if pval == 'not': + return split_penalty.UNBREAKABLE + + if cur_token.node_split_penalty > 0: + return cur_token.node_split_penalty + + if style.Get('SPLIT_BEFORE_LOGICAL_OPERATOR'): + # Prefer to split before 'and' and 'or'. + if pval in _LOGICAL_OPERATORS: + return style.Get('SPLIT_PENALTY_LOGICAL_OPERATOR') + if cval in _LOGICAL_OPERATORS: + return 0 + else: + # Prefer to split after 'and' and 'or'. + if pval in _LOGICAL_OPERATORS: + return 0 + if cval in _LOGICAL_OPERATORS: + return style.Get('SPLIT_PENALTY_LOGICAL_OPERATOR') + + if style.Get('SPLIT_BEFORE_BITWISE_OPERATOR'): + # Prefer to split before '&', '|', and '^'. + if pval in _BITWISE_OPERATORS: + return style.Get('SPLIT_PENALTY_BITWISE_OPERATOR') + if cval in _BITWISE_OPERATORS: + return 0 + else: + # Prefer to split after '&', '|', and '^'. + if pval in _BITWISE_OPERATORS: + return 0 + if cval in _BITWISE_OPERATORS: + return style.Get('SPLIT_PENALTY_BITWISE_OPERATOR') + + if (format_token.Subtype.COMP_FOR in cur_token.subtypes or + format_token.Subtype.COMP_IF in cur_token.subtypes): + # We don't mind breaking before the 'for' or 'if' of a list comprehension. + return 0 + if format_token.Subtype.UNARY_OPERATOR in prev_token.subtypes: + # Try not to break after a unary operator. + return style.Get('SPLIT_PENALTY_AFTER_UNARY_OPERATOR') + if pval == ',': + # Breaking after a comma is fine, if need be. + return 0 + if prev_token.is_binary_op: + # We would rather not split after an equality operator. + return 20 + if (format_token.Subtype.VARARGS_STAR in prev_token.subtypes or + format_token.Subtype.KWARGS_STAR_STAR in prev_token.subtypes): + # Don't split after a varargs * or kwargs **. + return split_penalty.UNBREAKABLE + if prev_token.OpensScope() and cval != '(': + # Slightly prefer + return style.Get('SPLIT_PENALTY_AFTER_OPENING_BRACKET') + if cval == ':': + # Don't split before a colon. + return split_penalty.UNBREAKABLE + if cval == '=': + # Don't split before an assignment. + return split_penalty.UNBREAKABLE + if (format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in prev_token.subtypes or + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN in cur_token.subtypes): + # Don't break before or after an default or named assignment. + return split_penalty.UNBREAKABLE + if cval == '==': + # We would rather not split before an equality operator. + return split_penalty.STRONGLY_CONNECTED + if cur_token.ClosesScope(): + # Give a slight penalty for splitting before the closing scope. + return 100 + if pval in _TERM_OPERATORS or cval in _TERM_OPERATORS: + return 50 + return 0 diff --git a/yapf/yapflib/verifier.py b/yapf/yapflib/verifier.py new file mode 100644 index 0000000..bcbe6fb --- /dev/null +++ b/yapf/yapflib/verifier.py @@ -0,0 +1,93 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Verify that the generated code is valid code. + +This takes a line of code and "normalizes" it. I.e., it transforms the snippet +into something that has the potential to compile. + + VerifyCode(): the main function exported by this module. +""" + +import ast +import re +import sys +import textwrap + + +class InternalError(Exception): + """Internal error in verifying formatted code.""" + pass + + +def VerifyCode(code): + """Verify that the reformatted code is syntactically correct. + + Arguments: + code: (unicode) The reformatted code snippet. + + Raises: + SyntaxError if the code was reformatted incorrectly. + """ + try: + compile(textwrap.dedent(code).encode('UTF-8'), '<string>', 'exec') + except SyntaxError: + try: + ast.parse(textwrap.dedent(code.lstrip('\n')).lstrip(), '<string>', 'exec') + except SyntaxError: + try: + normalized_code = _NormalizeCode(code) + compile(normalized_code.encode('UTF-8'), '<string>', 'exec') + except SyntaxError: + raise InternalError(sys.exc_info()[1]) + + +def _NormalizeCode(code): + """Make sure that the code snippet is compilable.""" + code = textwrap.dedent(code.lstrip('\n')).lstrip() + + # Split the code to lines and get rid of all leading full-comment lines as + # they can mess up the normalization attempt. + lines = code.split('\n') + i = 0 + for i, line in enumerate(lines): + line = line.strip() + if line and not line.startswith('#'): + break + code = '\n'.join(lines[i:]) + '\n' + + if re.match(r'(if|while|for|with|def|class|async|await)\b', code): + code += '\n pass' + elif re.match(r'(elif|else)\b', code): + try: + try_code = 'if True:\n pass\n' + code + '\n pass' + ast.parse( + textwrap.dedent(try_code.lstrip('\n')).lstrip(), '<string>', 'exec') + code = try_code + except SyntaxError: + # The assumption here is that the code is on a single line. + code = 'if True: pass\n' + code + elif code.startswith('@'): + code += '\ndef _():\n pass' + elif re.match(r'try\b', code): + code += '\n pass\nexcept:\n pass' + elif re.match(r'(except|finally)\b', code): + code = 'try:\n pass\n' + code + '\n pass' + elif re.match(r'(return|yield)\b', code): + code = 'def _():\n ' + code + elif re.match(r'(continue|break)\b', code): + code = 'while True:\n ' + code + elif re.match(r'print\b', code): + code = 'from __future__ import print_function\n' + code + + return code + '\n' diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py new file mode 100644 index 0000000..282dea3 --- /dev/null +++ b/yapf/yapflib/yapf_api.py @@ -0,0 +1,292 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Entry points for YAPF. + +The main APIs that YAPF exposes to drive the reformatting. + + FormatFile(): reformat a file. + FormatCode(): reformat a string of code. + +These APIs have some common arguments: + + style_config: (string) Either a style name or a path to a file that contains + formatting style settings. If None is specified, use the default style + as set in style.DEFAULT_STYLE_FACTORY + lines: (list of tuples of integers) A list of tuples of lines, [start, end], + that we want to format. The lines are 1-based indexed. It can be used by + third-party code (e.g., IDEs) when reformatting a snippet of code rather + than a whole file. + print_diff: (bool) Instead of returning the reformatted source, return a + diff that turns the formatted source into reformatter source. + verify: (bool) True if reformatted code should be verified for syntax. +""" + +import difflib +import re +import sys + +from lib2to3.pgen2 import parse + +from yapf.yapflib import blank_line_calculator +from yapf.yapflib import comment_splicer +from yapf.yapflib import continuation_splicer +from yapf.yapflib import file_resources +from yapf.yapflib import identify_container +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_unwrapper +from yapf.yapflib import pytree_utils +from yapf.yapflib import reformatter +from yapf.yapflib import split_penalty +from yapf.yapflib import style +from yapf.yapflib import subtype_assigner + + +def FormatFile(filename, + style_config=None, + lines=None, + print_diff=False, + verify=False, + in_place=False, + logger=None): + """Format a single Python file and return the formatted code. + + Arguments: + filename: (unicode) The file to reformat. + in_place: (bool) If True, write the reformatted code back to the file. + logger: (io streamer) A stream to output logging. + remaining arguments: see comment at the top of this module. + + Returns: + Tuple of (reformatted_code, encoding, changed). reformatted_code is None if + the file is successfully written to (having used in_place). reformatted_code + is a diff if print_diff is True. + + Raises: + IOError: raised if there was an error reading the file. + ValueError: raised if in_place and print_diff are both specified. + """ + _CheckPythonVersion() + + if in_place and print_diff: + raise ValueError('Cannot pass both in_place and print_diff.') + + original_source, newline, encoding = ReadFile(filename, logger) + reformatted_source, changed = FormatCode( + original_source, + style_config=style_config, + filename=filename, + lines=lines, + print_diff=print_diff, + verify=verify) + if reformatted_source.rstrip('\n'): + lines = reformatted_source.rstrip('\n').split('\n') + reformatted_source = newline.join(line for line in lines) + newline + if in_place: + if original_source and original_source != reformatted_source: + file_resources.WriteReformattedCode(filename, reformatted_source, + encoding, in_place) + return None, encoding, changed + + return reformatted_source, encoding, changed + + +def FormatCode(unformatted_source, + filename='<unknown>', + style_config=None, + lines=None, + print_diff=False, + verify=False): + """Format a string of Python code. + + This provides an alternative entry point to YAPF. + + Arguments: + unformatted_source: (unicode) The code to format. + filename: (unicode) The name of the file being reformatted. + remaining arguments: see comment at the top of this module. + + Returns: + Tuple of (reformatted_source, changed). reformatted_source conforms to the + desired formatting style. changed is True if the source changed. + """ + _CheckPythonVersion() + style.SetGlobalStyle(style.CreateStyleFromConfig(style_config)) + if not unformatted_source.endswith('\n'): + unformatted_source += '\n' + + try: + tree = pytree_utils.ParseCodeToTree(unformatted_source) + except parse.ParseError as e: + e.msg = filename + ': ' + e.msg + raise + + # Run passes on the tree, modifying it in place. + comment_splicer.SpliceComments(tree) + continuation_splicer.SpliceContinuations(tree) + subtype_assigner.AssignSubtypes(tree) + identify_container.IdentifyContainers(tree) + split_penalty.ComputeSplitPenalties(tree) + blank_line_calculator.CalculateBlankLines(tree) + + uwlines = pytree_unwrapper.UnwrapPyTree(tree) + for uwl in uwlines: + uwl.CalculateFormattingInformation() + + lines = _LineRangesToSet(lines) + _MarkLinesToFormat(uwlines, lines) + reformatted_source = reformatter.Reformat( + _SplitSemicolons(uwlines), verify, lines) + + if unformatted_source == reformatted_source: + return '' if print_diff else reformatted_source, False + + code_diff = _GetUnifiedDiff( + unformatted_source, reformatted_source, filename=filename) + + if print_diff: + return code_diff, code_diff.strip() != '' # pylint: disable=g-explicit-bool-comparison + + return reformatted_source, True + + +def _CheckPythonVersion(): # pragma: no cover + errmsg = 'yapf is only supported for Python 2.7 or 3.4+' + if sys.version_info[0] == 2: + if sys.version_info[1] < 7: + raise RuntimeError(errmsg) + elif sys.version_info[0] == 3: + if sys.version_info[1] < 4: + raise RuntimeError(errmsg) + + +def ReadFile(filename, logger=None): + """Read the contents of the file. + + An optional logger can be specified to emit messages to your favorite logging + stream. If specified, then no exception is raised. This is external so that it + can be used by third-party applications. + + Arguments: + filename: (unicode) The name of the file. + logger: (function) A function or lambda that takes a string and emits it. + + Returns: + The contents of filename. + + Raises: + IOError: raised if there was an error reading the file. + """ + try: + encoding = file_resources.FileEncoding(filename) + + # Preserves line endings. + with py3compat.open_with_encoding( + filename, mode='r', encoding=encoding, newline='') as fd: + lines = fd.readlines() + + line_ending = file_resources.LineEnding(lines) + source = '\n'.join(line.rstrip('\r\n') for line in lines) + '\n' + return source, line_ending, encoding + except IOError as err: # pragma: no cover + if logger: + logger(err) + raise + + +def _SplitSemicolons(uwlines): + res = [] + for uwline in uwlines: + res.extend(uwline.Split()) + return res + + +DISABLE_PATTERN = r'^#.*\byapf:\s*disable\b' +ENABLE_PATTERN = r'^#.*\byapf:\s*enable\b' + + +def _LineRangesToSet(line_ranges): + """Return a set of lines in the range.""" + + if line_ranges is None: + return None + + line_set = set() + for low, high in sorted(line_ranges): + line_set.update(range(low, high + 1)) + + return line_set + + +def _MarkLinesToFormat(uwlines, lines): + """Skip sections of code that we shouldn't reformat.""" + if lines: + for uwline in uwlines: + uwline.disable = not lines.intersection( + range(uwline.lineno, uwline.last.lineno + 1)) + + # Now go through the lines and disable any lines explicitly marked as + # disabled. + index = 0 + while index < len(uwlines): + uwline = uwlines[index] + if uwline.is_comment: + if _DisableYAPF(uwline.first.value.strip()): + index += 1 + while index < len(uwlines): + uwline = uwlines[index] + if uwline.is_comment and _EnableYAPF(uwline.first.value.strip()): + break + uwline.disable = True + index += 1 + elif re.search(DISABLE_PATTERN, uwline.last.value.strip(), re.IGNORECASE): + uwline.disable = True + index += 1 + + +def _DisableYAPF(line): + return (re.search(DISABLE_PATTERN, + line.split('\n')[0].strip(), re.IGNORECASE) or + re.search(DISABLE_PATTERN, + line.split('\n')[-1].strip(), re.IGNORECASE)) + + +def _EnableYAPF(line): + return (re.search(ENABLE_PATTERN, + line.split('\n')[0].strip(), re.IGNORECASE) or + re.search(ENABLE_PATTERN, + line.split('\n')[-1].strip(), re.IGNORECASE)) + + +def _GetUnifiedDiff(before, after, filename='code'): + """Get a unified diff of the changes. + + Arguments: + before: (unicode) The original source code. + after: (unicode) The reformatted source code. + filename: (unicode) The code's filename. + + Returns: + The unified diff text. + """ + before = before.splitlines() + after = after.splitlines() + return '\n'.join( + difflib.unified_diff( + before, + after, + filename, + filename, + '(original)', + '(reformatted)', + lineterm='')) + '\n' diff --git a/yapftests/__init__.py b/yapftests/__init__.py new file mode 100644 index 0000000..e7522b2 --- /dev/null +++ b/yapftests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/yapftests/blank_line_calculator_test.py b/yapftests/blank_line_calculator_test.py new file mode 100644 index 0000000..2a27a2f --- /dev/null +++ b/yapftests/blank_line_calculator_test.py @@ -0,0 +1,355 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.blank_line_calculator.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style +from yapf.yapflib import yapf_api + +from yapftests import yapf_test_helper + + +class BasicBlankLineCalculatorTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testDecorators(self): + unformatted_code = textwrap.dedent("""\ + @bork() + + def foo(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + @bork() + def foo(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testComplexDecorators(self): + unformatted_code = textwrap.dedent("""\ + import sys + @bork() + + def foo(): + pass + @fork() + + class moo(object): + @bar() + @baz() + + def method(self): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import sys + + + @bork() + def foo(): + pass + + + @fork() + class moo(object): + + @bar() + @baz() + def method(self): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testCodeAfterFunctionsAndClasses(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + pass + top_level_code = True + class moo(object): + def method_1(self): + pass + ivar_a = 42 + ivar_b = 13 + def method_2(self): + pass + try: + raise Error + except Error as error: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + pass + + + top_level_code = True + + + class moo(object): + + def method_1(self): + pass + + ivar_a = 42 + ivar_b = 13 + + def method_2(self): + pass + + + try: + raise Error + except Error as error: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testCommentSpacing(self): + unformatted_code = textwrap.dedent("""\ + # This is the first comment + # And it's multiline + + # This is the second comment + + def foo(): + pass + + # multiline before a + # class definition + + # This is the second comment + + class qux(object): + pass + + + # An attached comment. + class bar(object): + '''class docstring''' + # Comment attached to + # function + def foo(self): + '''Another docstring.''' + # Another multiline + # comment + pass + """) + expected_formatted_code = textwrap.dedent("""\ + # This is the first comment + # And it's multiline + + # This is the second comment + + + def foo(): + pass + + + # multiline before a + # class definition + + # This is the second comment + + + class qux(object): + pass + + + # An attached comment. + class bar(object): + '''class docstring''' + + # Comment attached to + # function + def foo(self): + '''Another docstring.''' + # Another multiline + # comment + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testCommentBeforeMethod(self): + code = textwrap.dedent("""\ + class foo(object): + + # pylint: disable=invalid-name + def f(self): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentsBeforeClassDefs(self): + code = textwrap.dedent('''\ + """Test.""" + + # Comment + + + class Foo(object): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentsBeforeDecorator(self): + code = textwrap.dedent("""\ + # The @foo operator adds bork to a(). + @foo() + def a(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + # Hello world + + + @foo() + def a(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentsAfterDecorator(self): + code = textwrap.dedent("""\ + class _(): + + def _(): + pass + + @pytest.mark.xfail(reason="#709 and #710") + # also + #@pytest.mark.xfail(setuptools.tests.is_ascii, + # reason="https://github.com/pypa/setuptools/issues/706") + def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testInnerClasses(self): + unformatted_code = textwrap.dedent("""\ + class DeployAPIClient(object): + class Error(Exception): pass + + class TaskValidationError(Error): pass + + class DeployAPIHTTPError(Error): pass + """) + expected_formatted_code = textwrap.dedent("""\ + class DeployAPIClient(object): + + class Error(Exception): + pass + + class TaskValidationError(Error): + pass + + class DeployAPIHTTPError(Error): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testLinesOnRangeBoundary(self): + unformatted_code = textwrap.dedent(u"""\ + def A(): + pass + + def B(): # 4 + pass # 5 + + def C(): + pass + def D(): # 9 + pass # 10 + def E(): + pass + """) + expected_formatted_code = textwrap.dedent(u"""\ + def A(): + pass + + + def B(): # 4 + pass # 5 + + + def C(): + pass + + + def D(): # 9 + pass # 10 + + + def E(): + pass + """) + code, changed = yapf_api.FormatCode( + unformatted_code, lines=[(4, 5), (9, 10)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertTrue(changed) + + def testLinesRangeBoundaryNotOutside(self): + unformatted_code = textwrap.dedent(u"""\ + def A(): + pass + + + + def B(): # 6 + pass # 7 + + + + def C(): + pass + """) + expected_formatted_code = textwrap.dedent(u"""\ + def A(): + pass + + + + def B(): # 6 + pass # 7 + + + + def C(): + pass + """) + code, changed = yapf_api.FormatCode(unformatted_code, lines=[(6, 7)]) + self.assertCodeEqual(expected_formatted_code, code) + self.assertFalse(changed) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/comment_splicer_test.py b/yapftests/comment_splicer_test.py new file mode 100644 index 0000000..aacc888 --- /dev/null +++ b/yapftests/comment_splicer_test.py @@ -0,0 +1,334 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.comment_splicer.""" + +import textwrap +import unittest + +from yapf.yapflib import comment_splicer +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils + + +class CommentSplicerTest(unittest.TestCase): + + def _AssertNodeType(self, expected_type, node): + self.assertEqual(expected_type, pytree_utils.NodeName(node)) + + def _AssertNodeIsComment(self, node, text_in_comment=None): + if pytree_utils.NodeName(node) == 'simple_stmt': + self._AssertNodeType('COMMENT', node.children[0]) + node_value = node.children[0].value + else: + self._AssertNodeType('COMMENT', node) + node_value = node.value + if text_in_comment is not None: + self.assertIn(text_in_comment, node_value) + + def _FindNthChildNamed(self, node, name, n=1): + for i, child in enumerate( + py3compat.ifilter(lambda c: pytree_utils.NodeName(c) == name, + node.pre_order())): + if i == n - 1: + return child + raise RuntimeError('No Nth child for n={0}'.format(n)) + + def testSimpleInline(self): + code = 'foo = 1 # and a comment\n' + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + expr = tree.children[0].children[0] + # Check that the expected node is still expr_stmt, but now it has 4 children + # (before comment splicing it had 3), the last child being the comment. + self._AssertNodeType('expr_stmt', expr) + self.assertEqual(4, len(expr.children)) + comment_node = expr.children[3] + self._AssertNodeIsComment(comment_node, '# and a comment') + + def testSimpleSeparateLine(self): + code = textwrap.dedent(r''' + foo = 1 + # first comment + bar = 2 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # The comment should've been added to the root's children (now 4, including + # the ENDMARKER in the end. + self.assertEqual(4, len(tree.children)) + comment_node = tree.children[1] + self._AssertNodeIsComment(comment_node) + + def testTwoLineComment(self): + code = textwrap.dedent(r''' + foo = 1 + # first comment + # second comment + bar = 2 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # This is similar to the single-line standalone comment. + self.assertEqual(4, len(tree.children)) + self._AssertNodeIsComment(tree.children[1]) + + def testCommentIsFirstChildInCompound(self): + code = textwrap.dedent(r''' + if x: + # a comment + foo = 1 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # Look into the suite node under the 'if'. We don't care about the NEWLINE + # leaf but the new COMMENT must be a child of the suite and before the + # actual code leaf. + if_suite = tree.children[0].children[3] + self._AssertNodeType('NEWLINE', if_suite.children[0]) + self._AssertNodeIsComment(if_suite.children[1]) + + def testCommentIsLastChildInCompound(self): + code = textwrap.dedent(r''' + if x: + foo = 1 + # a comment + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # Look into the suite node under the 'if'. We don't care about the DEDENT + # leaf but the new COMMENT must be a child of the suite and after the + # actual code leaf. + if_suite = tree.children[0].children[3] + self._AssertNodeType('DEDENT', if_suite.children[-1]) + self._AssertNodeIsComment(if_suite.children[-2]) + + def testInlineAfterSeparateLine(self): + code = textwrap.dedent(r''' + bar = 1 + # line comment + foo = 1 # inline comment + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # The separate line comment should become a child of the root, while + # the inline comment remains within its simple_node. + sep_comment_node = tree.children[1] + self._AssertNodeIsComment(sep_comment_node, '# line comment') + + expr = tree.children[2].children[0] + inline_comment_node = expr.children[-1] + self._AssertNodeIsComment(inline_comment_node, '# inline comment') + + def testSeparateLineAfterInline(self): + code = textwrap.dedent(r''' + bar = 1 + foo = 1 # inline comment + # line comment + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # The separate line comment should become a child of the root, while + # the inline comment remains within its simple_node. + sep_comment_node = tree.children[-2] + self._AssertNodeIsComment(sep_comment_node, '# line comment') + + expr = tree.children[1].children[0] + inline_comment_node = expr.children[-1] + self._AssertNodeIsComment(inline_comment_node, '# inline comment') + + def testCommentBeforeDedent(self): + code = textwrap.dedent(r''' + if bar: + z = 1 + # a comment + j = 2 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # The comment should go under the tree root, not under the 'if'. + self._AssertNodeIsComment(tree.children[1]) + if_suite = tree.children[0].children[3] + self._AssertNodeType('DEDENT', if_suite.children[-1]) + + def testCommentBeforeDedentTwoLevel(self): + code = textwrap.dedent(r''' + if foo: + if bar: + z = 1 + # a comment + y = 1 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + if_suite = tree.children[0].children[3] + # The comment is in the first if_suite, not the nested if under it. It's + # right before the DEDENT + self._AssertNodeIsComment(if_suite.children[-2]) + self._AssertNodeType('DEDENT', if_suite.children[-1]) + + def testCommentBeforeDedentTwoLevelImproperlyIndented(self): + code = textwrap.dedent(r''' + if foo: + if bar: + z = 1 + # comment 2 + y = 1 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # The comment here is indented by 3 spaces, which is unlike any of the + # surrounding statement indentation levels. The splicer attaches it to the + # "closest" parent with smaller indentation. + if_suite = tree.children[0].children[3] + # The comment is in the first if_suite, not the nested if under it. It's + # right before the DEDENT + self._AssertNodeIsComment(if_suite.children[-2]) + self._AssertNodeType('DEDENT', if_suite.children[-1]) + + def testCommentBeforeDedentThreeLevel(self): + code = textwrap.dedent(r''' + if foo: + if bar: + z = 1 + # comment 2 + # comment 1 + # comment 0 + j = 2 + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + # comment 0 should go under the tree root + self._AssertNodeIsComment(tree.children[1], '# comment 0') + + # comment 1 is in the first if_suite, right before the DEDENT + if_suite_1 = self._FindNthChildNamed(tree, 'suite', n=1) + self._AssertNodeIsComment(if_suite_1.children[-2], '# comment 1') + self._AssertNodeType('DEDENT', if_suite_1.children[-1]) + + # comment 2 is in if_suite nested under the first if suite, + # right before the DEDENT + if_suite_2 = self._FindNthChildNamed(tree, 'suite', n=2) + self._AssertNodeIsComment(if_suite_2.children[-2], '# comment 2') + self._AssertNodeType('DEDENT', if_suite_2.children[-1]) + + def testCommentsInClass(self): + code = textwrap.dedent(r''' + class Foo: + """docstring abc...""" + # top-level comment + def foo(): pass + # another comment + ''') + + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + class_suite = tree.children[0].children[3] + another_comment = class_suite.children[-2] + self._AssertNodeIsComment(another_comment, '# another') + + # It's OK for the comment to be a child of funcdef, as long as it's + # the first child and thus comes before the 'def'. + funcdef = class_suite.children[3] + toplevel_comment = funcdef.children[0] + self._AssertNodeIsComment(toplevel_comment, '# top-level') + + def testMultipleBlockComments(self): + code = textwrap.dedent(r''' + # Block comment number 1 + + # Block comment number 2 + def f(): + pass + ''') + + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + funcdef = tree.children[0] + block_comment_1 = funcdef.children[0] + self._AssertNodeIsComment(block_comment_1, '# Block comment number 1') + + block_comment_2 = funcdef.children[1] + self._AssertNodeIsComment(block_comment_2, '# Block comment number 2') + + def testCommentsOnDedents(self): + code = textwrap.dedent(r''' + class Foo(object): + # A comment for qux. + def qux(self): + pass + + # Interim comment. + + def mux(self): + pass + ''') + + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + classdef = tree.children[0] + class_suite = classdef.children[6] + qux_comment = class_suite.children[1] + self._AssertNodeIsComment(qux_comment, '# A comment for qux.') + + interim_comment = class_suite.children[4] + self._AssertNodeIsComment(interim_comment, '# Interim comment.') + + def testExprComments(self): + code = textwrap.dedent(r''' + foo( # Request fractions of an hour. + 948.0/3600, 20) + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + trailer = self._FindNthChildNamed(tree, 'trailer', 1) + comment = trailer.children[1] + self._AssertNodeIsComment(comment, '# Request fractions of an hour.') + + def testMultipleCommentsInOneExpr(self): + code = textwrap.dedent(r''' + foo( # com 1 + 948.0/3600, # com 2 + 20 + 12 # com 3 + ) + ''') + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + + trailer = self._FindNthChildNamed(tree, 'trailer', 1) + self._AssertNodeIsComment(trailer.children[1], '# com 1') + + arglist = self._FindNthChildNamed(tree, 'arglist', 1) + self._AssertNodeIsComment(arglist.children[2], '# com 2') + + arith_expr = self._FindNthChildNamed(tree, 'arith_expr', 1) + self._AssertNodeIsComment(arith_expr.children[-1], '# com 3') + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/file_resources_test.py b/yapftests/file_resources_test.py new file mode 100644 index 0000000..70cea46 --- /dev/null +++ b/yapftests/file_resources_test.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.file_resources.""" + +import contextlib +import os +import shutil +import tempfile +import unittest + +from yapf.yapflib import errors +from yapf.yapflib import file_resources +from yapf.yapflib import py3compat + +from yapftests import utils + + +@contextlib.contextmanager +def _restore_working_dir(): + curdir = os.getcwd() + try: + yield + finally: + os.chdir(curdir) + + +class GetDefaultStyleForDirTest(unittest.TestCase): + + def setUp(self): + self.test_tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_tmpdir) + + def test_no_local_style(self): + test_file = os.path.join(self.test_tmpdir, 'file.py') + style_name = file_resources.GetDefaultStyleForDir(test_file) + self.assertEqual(style_name, 'pep8') + + def test_with_local_style(self): + # Create an empty .style.yapf file in test_tmpdir + style_file = os.path.join(self.test_tmpdir, '.style.yapf') + open(style_file, 'w').close() + + test_filename = os.path.join(self.test_tmpdir, 'file.py') + self.assertEqual(style_file, + file_resources.GetDefaultStyleForDir(test_filename)) + + test_filename = os.path.join(self.test_tmpdir, 'dir1', 'file.py') + self.assertEqual(style_file, + file_resources.GetDefaultStyleForDir(test_filename)) + + +def _touch_files(filenames): + for name in filenames: + open(name, 'a').close() + + +class GetCommandLineFilesTest(unittest.TestCase): + + def setUp(self): + self.test_tmpdir = tempfile.mkdtemp() + self.old_dir = os.getcwd() + + def tearDown(self): + shutil.rmtree(self.test_tmpdir) + os.chdir(self.old_dir) + + def _make_test_dir(self, name): + fullpath = os.path.normpath(os.path.join(self.test_tmpdir, name)) + os.makedirs(fullpath) + return fullpath + + def test_find_files_not_dirs(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test2') + file1 = os.path.join(tdir1, 'testfile1.py') + file2 = os.path.join(tdir2, 'testfile2.py') + _touch_files([file1, file2]) + + self.assertEqual( + file_resources.GetCommandLineFiles( + [file1, file2], recursive=False, exclude=None), [file1, file2]) + self.assertEqual( + file_resources.GetCommandLineFiles( + [file1, file2], recursive=True, exclude=None), [file1, file2]) + + def test_nonrecursive_find_in_dir(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test1/foo') + file1 = os.path.join(tdir1, 'testfile1.py') + file2 = os.path.join(tdir2, 'testfile2.py') + _touch_files([file1, file2]) + + self.assertRaises( + errors.YapfError, + file_resources.GetCommandLineFiles, + command_line_file_list=[tdir1], + recursive=False, + exclude=None) + + def test_recursive_find_in_dir(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test2/testinner/') + tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + self.assertEqual( + sorted( + file_resources.GetCommandLineFiles( + [self.test_tmpdir], recursive=True, exclude=None)), + sorted(files)) + + def test_recursive_find_in_dir_with_exclude(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test2/testinner/') + tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + self.assertEqual( + sorted( + file_resources.GetCommandLineFiles( + [self.test_tmpdir], recursive=True, exclude=['*test*3.py'])), + sorted([ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + ])) + + def test_find_with_excluded_hidden_dirs(self): + tdir1 = self._make_test_dir('.test1') + tdir2 = self._make_test_dir('test_2') + tdir3 = self._make_test_dir('test.3') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + actual = file_resources.GetCommandLineFiles( + [self.test_tmpdir], recursive=True, exclude=['*.test1*']) + + self.assertEqual( + sorted(actual), + sorted([ + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ])) + + def test_find_with_excluded_hidden_dirs_relative(self): + """Test find with excluded hidden dirs. + + A regression test against a specific case where a hidden directory (one + beginning with a period) is being excluded, but it is also an immediate + child of the current directory which has been specified in a relative + manner. + + At its core, the bug has to do with overzelous stripping of "./foo" so that + it removes too much from "./.foo" . + """ + tdir1 = self._make_test_dir('.test1') + tdir2 = self._make_test_dir('test_2') + tdir3 = self._make_test_dir('test.3') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + # We must temporarily change the current directory, so that we test against + # patterns like ./.test1/file instead of /tmp/foo/.test1/file + with _restore_working_dir(): + + os.chdir(self.test_tmpdir) + actual = file_resources.GetCommandLineFiles( + [os.path.relpath(self.test_tmpdir)], + recursive=True, + exclude=['*.test1*']) + + self.assertEqual( + sorted(actual), + sorted([ + os.path.join( + os.path.relpath(self.test_tmpdir), os.path.basename(tdir2), + 'testfile2.py'), + os.path.join( + os.path.relpath(self.test_tmpdir), os.path.basename(tdir3), + 'testfile3.py'), + ])) + + def test_find_with_excluded_dirs(self): + tdir1 = self._make_test_dir('test1') + tdir2 = self._make_test_dir('test2/testinner/') + tdir3 = self._make_test_dir('test3/foo/bar/bas/xxx') + files = [ + os.path.join(tdir1, 'testfile1.py'), + os.path.join(tdir2, 'testfile2.py'), + os.path.join(tdir3, 'testfile3.py'), + ] + _touch_files(files) + + os.chdir(self.test_tmpdir) + + found = sorted( + file_resources.GetCommandLineFiles( + ['test1', 'test2', 'test3'], + recursive=True, + exclude=[ + 'test1', + 'test2/testinner/', + ])) + + self.assertEqual(found, ['test3/foo/bar/bas/xxx/testfile3.py']) + + found = sorted( + file_resources.GetCommandLineFiles( + ['.'], recursive=True, exclude=[ + 'test1', + 'test3', + ])) + + self.assertEqual(found, ['./test2/testinner/testfile2.py']) + + def test_find_with_excluded_current_dir(self): + with self.assertRaises(errors.YapfError): + file_resources.GetCommandLineFiles([], False, exclude=['./z']) + + +class IsPythonFileTest(unittest.TestCase): + + def setUp(self): + self.test_tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_tmpdir) + + def test_with_py_extension(self): + file1 = os.path.join(self.test_tmpdir, 'testfile1.py') + self.assertTrue(file_resources.IsPythonFile(file1)) + + def test_empty_without_py_extension(self): + file1 = os.path.join(self.test_tmpdir, 'testfile1') + self.assertFalse(file_resources.IsPythonFile(file1)) + file2 = os.path.join(self.test_tmpdir, 'testfile1.rb') + self.assertFalse(file_resources.IsPythonFile(file2)) + + def test_python_shebang(self): + file1 = os.path.join(self.test_tmpdir, 'testfile1') + with open(file1, 'w') as f: + f.write(u'#!/usr/bin/python\n') + self.assertTrue(file_resources.IsPythonFile(file1)) + + file2 = os.path.join(self.test_tmpdir, 'testfile2.run') + with open(file2, 'w') as f: + f.write(u'#! /bin/python2\n') + self.assertTrue(file_resources.IsPythonFile(file1)) + + def test_with_latin_encoding(self): + file1 = os.path.join(self.test_tmpdir, 'testfile1') + with py3compat.open_with_encoding(file1, mode='w', encoding='latin-1') as f: + f.write(u'#! /bin/python2\n') + self.assertTrue(file_resources.IsPythonFile(file1)) + + def test_with_invalid_encoding(self): + file1 = os.path.join(self.test_tmpdir, 'testfile1') + with open(file1, 'w') as f: + f.write(u'#! /bin/python2\n') + f.write(u'# -*- coding: iso-3-14159 -*-\n') + self.assertFalse(file_resources.IsPythonFile(file1)) + + +class IsIgnoredTest(unittest.TestCase): + + def test_root_path(self): + self.assertTrue(file_resources.IsIgnored('media', ['media'])) + self.assertFalse(file_resources.IsIgnored('media', ['media/*'])) + + def test_sub_path(self): + self.assertTrue(file_resources.IsIgnored('media/a', ['*/a'])) + self.assertTrue(file_resources.IsIgnored('media/b', ['media/*'])) + self.assertTrue(file_resources.IsIgnored('media/b/c', ['*/*/c'])) + + def test_trailing_slash(self): + self.assertTrue(file_resources.IsIgnored('z', ['z'])) + self.assertTrue(file_resources.IsIgnored('z', ['z/'])) + + +class BufferedByteStream(object): + + def __init__(self): + self.stream = py3compat.BytesIO() + + def getvalue(self): # pylint: disable=invalid-name + return self.stream.getvalue().decode('utf-8') + + @property + def buffer(self): + return self.stream + + +class WriteReformattedCodeTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.test_tmpdir = tempfile.mkdtemp() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.test_tmpdir) + + def test_write_to_file(self): + s = u'foobar\n' + with utils.NamedTempFile(dirname=self.test_tmpdir) as (f, fname): + file_resources.WriteReformattedCode( + fname, s, in_place=True, encoding='utf-8') + f.flush() + + with open(fname) as f2: + self.assertEqual(f2.read(), s) + + def test_write_to_stdout(self): + s = u'foobar' + stream = BufferedByteStream() if py3compat.PY3 else py3compat.StringIO() + with utils.stdout_redirector(stream): + file_resources.WriteReformattedCode( + None, s, in_place=False, encoding='utf-8') + self.assertEqual(stream.getvalue(), s) + + def test_write_encoded_to_stdout(self): + s = '\ufeff# -*- coding: utf-8 -*-\nresult = "passed"\n' # pylint: disable=anomalous-unicode-escape-in-string + stream = BufferedByteStream() if py3compat.PY3 else py3compat.StringIO() + with utils.stdout_redirector(stream): + file_resources.WriteReformattedCode( + None, s, in_place=False, encoding='utf-8') + self.assertEqual(stream.getvalue(), s) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/format_decision_state_test.py b/yapftests/format_decision_state_test.py new file mode 100644 index 0000000..6122960 --- /dev/null +++ b/yapftests/format_decision_state_test.py @@ -0,0 +1,145 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.format_decision_state.""" + +import textwrap +import unittest + +from yapf.yapflib import format_decision_state +from yapf.yapflib import pytree_utils +from yapf.yapflib import style +from yapf.yapflib import unwrapped_line + +from yapftests import yapf_test_helper + + +class FormatDecisionStateTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testSimpleFunctionDefWithNoSplitting(self): + code = textwrap.dedent(r""" + def f(a, b): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + uwline = unwrapped_line.UnwrappedLine(0, _FilterLine(uwlines[0])) + uwline.CalculateFormattingInformation() + + # Add: 'f' + state = format_decision_state.FormatDecisionState(uwline, 0) + state.MoveStateToNextToken() + self.assertEqual('f', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + + # Add: '(' + state.AddTokenToState(False, True) + self.assertEqual('(', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + # Add: 'a' + state.AddTokenToState(False, True) + self.assertEqual('a', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + # Add: ',' + state.AddTokenToState(False, True) + self.assertEqual(',', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + # Add: 'b' + state.AddTokenToState(False, True) + self.assertEqual('b', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + # Add: ')' + state.AddTokenToState(False, True) + self.assertEqual(')', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + # Add: ':' + state.AddTokenToState(False, True) + self.assertEqual(':', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + self.assertFalse(state.MustSplit()) + + clone = state.Clone() + self.assertEqual(repr(state), repr(clone)) + + def testSimpleFunctionDefWithSplitting(self): + code = textwrap.dedent(r""" + def f(a, b): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + uwline = unwrapped_line.UnwrappedLine(0, _FilterLine(uwlines[0])) + uwline.CalculateFormattingInformation() + + # Add: 'f' + state = format_decision_state.FormatDecisionState(uwline, 0) + state.MoveStateToNextToken() + self.assertEqual('f', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + + # Add: '(' + state.AddTokenToState(True, True) + self.assertEqual('(', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + + # Add: 'a' + state.AddTokenToState(True, True) + self.assertEqual('a', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + + # Add: ',' + state.AddTokenToState(True, True) + self.assertEqual(',', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + + # Add: 'b' + state.AddTokenToState(True, True) + self.assertEqual('b', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + + # Add: ')' + state.AddTokenToState(True, True) + self.assertEqual(')', state.next_token.value) + self.assertTrue(state.CanSplit(False)) + + # Add: ':' + state.AddTokenToState(True, True) + self.assertEqual(':', state.next_token.value) + self.assertFalse(state.CanSplit(False)) + + clone = state.Clone() + self.assertEqual(repr(state), repr(clone)) + + +def _FilterLine(uwline): + """Filter out nonsemantic tokens from the UnwrappedLines.""" + return [ + ft for ft in uwline.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS + ] + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/format_token_test.py b/yapftests/format_token_test.py new file mode 100644 index 0000000..7dfb82a --- /dev/null +++ b/yapftests/format_token_test.py @@ -0,0 +1,86 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.format_token.""" + +import unittest + +from lib2to3 import pytree +from lib2to3.pgen2 import token + +from yapf.yapflib import format_token + + +class TabbedContinuationAlignPaddingTest(unittest.TestCase): + + def testSpace(self): + align_style = 'SPACE' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 2, 4) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 2, 4) + self.assertEqual(pad, ' ' * 2) + + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 2, 4) + self.assertEqual(pad, ' ' * 5) + + def testFixed(self): + align_style = 'FIXED' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4, 8) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4, 8) + self.assertEqual(pad, '\t' * 2) + + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4, 8) + self.assertEqual(pad, '\t' * 2) + + def testVAlignRight(self): + align_style = 'VALIGN-RIGHT' + + pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4, 8) + self.assertEqual(pad, '') + + pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4, 8) + self.assertEqual(pad, '\t') + + pad = format_token._TabbedContinuationAlignPadding(4, align_style, 4, 8) + self.assertEqual(pad, '\t') + + pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4, 8) + self.assertEqual(pad, '\t' * 2) + + +class FormatTokenTest(unittest.TestCase): + + def testSimple(self): + tok = format_token.FormatToken(pytree.Leaf(token.STRING, "'hello world'")) + self.assertEqual("FormatToken(name=STRING, value='hello world')", str(tok)) + self.assertTrue(tok.is_string) + + tok = format_token.FormatToken(pytree.Leaf(token.COMMENT, '# A comment')) + self.assertEqual('FormatToken(name=COMMENT, value=# A comment)', str(tok)) + self.assertTrue(tok.is_comment) + + def testIsMultilineString(self): + tok = format_token.FormatToken(pytree.Leaf(token.STRING, '"""hello"""')) + self.assertTrue(tok.is_multiline_string) + + tok = format_token.FormatToken(pytree.Leaf(token.STRING, 'r"""hello"""')) + self.assertTrue(tok.is_multiline_string) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/line_joiner_test.py b/yapftests/line_joiner_test.py new file mode 100644 index 0000000..6501bc8 --- /dev/null +++ b/yapftests/line_joiner_test.py @@ -0,0 +1,82 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.line_joiner.""" + +import textwrap +import unittest + +from yapf.yapflib import line_joiner +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class LineJoinerTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def _CheckLineJoining(self, code, join_lines): + """Check that the given UnwrappedLines are joined as expected. + + Arguments: + code: The code to check to see if we can join it. + join_lines: True if we expect the lines to be joined. + """ + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(line_joiner.CanMergeMultipleLines(uwlines), join_lines) + + def testSimpleSingleLineStatement(self): + code = textwrap.dedent(u"""\ + if isinstance(a, int): continue + """) + self._CheckLineJoining(code, join_lines=True) + + def testSimpleMultipleLineStatement(self): + code = textwrap.dedent(u"""\ + if isinstance(b, int): + continue + """) + self._CheckLineJoining(code, join_lines=False) + + def testSimpleMultipleLineComplexStatement(self): + code = textwrap.dedent(u"""\ + if isinstance(c, int): + while True: + continue + """) + self._CheckLineJoining(code, join_lines=False) + + def testSimpleMultipleLineStatementWithComment(self): + code = textwrap.dedent(u"""\ + if isinstance(d, int): continue # We're pleased that d's an int. + """) + self._CheckLineJoining(code, join_lines=True) + + def testSimpleMultipleLineStatementWithLargeIndent(self): + code = textwrap.dedent(u"""\ + if isinstance(e, int): continue + """) + self._CheckLineJoining(code, join_lines=True) + + def testOverColumnLimit(self): + code = textwrap.dedent(u"""\ + if instance(bbbbbbbbbbbbbbbbbbbbbbbbb, int): cccccccccccccccccccccccccc = ddddddddddddddddddddd + """) + self._CheckLineJoining(code, join_lines=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/main_test.py b/yapftests/main_test.py new file mode 100644 index 0000000..138853c --- /dev/null +++ b/yapftests/main_test.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.__init__.main.""" + +from contextlib import contextmanager +import sys +import unittest +import yapf + +from yapf.yapflib import py3compat + + +class IO(object): + """IO is a thin wrapper around StringIO. + + This is strictly to wrap the Python 3 StringIO object so that it can supply a + "buffer" attribute. + """ + + class Buffer(object): + + def __init__(self): + self.string_io = py3compat.StringIO() + + def write(self, s): + if py3compat.PY3 and isinstance(s, bytes): + s = str(s, 'utf-8') + self.string_io.write(s) + + def getvalue(self): + return self.string_io.getvalue() + + def __init__(self): + self.buffer = self.Buffer() + + def write(self, s): + self.buffer.write(s) + + def getvalue(self): + return self.buffer.getvalue() + + +@contextmanager +def captured_output(): + new_out, new_err = IO(), IO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + + +@contextmanager +def patched_input(code): + """Monkey patch code as though it were coming from stdin.""" + + def lines(): + for line in code.splitlines(): + yield line + raise EOFError() + + def patch_raw_input(lines=lines()): + return next(lines) + + try: + orig_raw_import = yapf.py3compat.raw_input + yapf.py3compat.raw_input = patch_raw_input + yield + finally: + yapf.py3compat.raw_input = orig_raw_import + + +class RunMainTest(unittest.TestCase): + + def testShouldHandleYapfError(self): + """run_main should handle YapfError and sys.exit(1).""" + expected_message = 'yapf: Input filenames did not match any python files\n' + sys.argv = ['yapf', 'foo.c'] + with captured_output() as (out, err): + with self.assertRaises(SystemExit): + yapf.run_main() + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), expected_message) + + +class MainTest(unittest.TestCase): + + def testNoPythonFilesMatched(self): + with self.assertRaisesRegexp(yapf.errors.YapfError, + 'did not match any python files'): + yapf.main(['yapf', 'foo.c']) + + def testEchoInput(self): + code = 'a = 1\nb = 2\n' + with patched_input(code): + with captured_output() as (out, _): + ret = yapf.main([]) + self.assertEqual(ret, 0) + self.assertEqual(out.getvalue(), code) + + def testEchoInputWithStyle(self): + code = 'def f(a = 1):\n return 2*a\n' + chromium_code = 'def f(a=1):\n return 2 * a\n' + with patched_input(code): + with captured_output() as (out, _): + ret = yapf.main(['-', '--style=chromium']) + self.assertEqual(ret, 0) + self.assertEqual(out.getvalue(), chromium_code) + + def testEchoBadInput(self): + bad_syntax = ' a = 1\n' + with patched_input(bad_syntax): + with captured_output() as (_, _): + with self.assertRaisesRegexp(SyntaxError, 'unexpected indent'): + yapf.main([]) + + def testHelp(self): + with captured_output() as (out, _): + ret = yapf.main(['-', '--style-help', '--style=pep8']) + self.assertEqual(ret, 0) + help_message = out.getvalue() + self.assertIn('indent_width=4', help_message) + self.assertIn('The number of spaces required before a trailing comment.', + help_message) + + def testVersion(self): + with captured_output() as (out, _): + ret = yapf.main(['-', '--version']) + self.assertEqual(ret, 0) + version = 'yapf {}\n'.format(yapf.__version__) + self.assertEqual(version, out.getvalue()) diff --git a/yapftests/pytree_unwrapper_test.py b/yapftests/pytree_unwrapper_test.py new file mode 100644 index 0000000..f95f366 --- /dev/null +++ b/yapftests/pytree_unwrapper_test.py @@ -0,0 +1,356 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.pytree_unwrapper.""" + +import textwrap +import unittest + +from yapf.yapflib import pytree_utils + +from yapftests import yapf_test_helper + + +class PytreeUnwrapperTest(yapf_test_helper.YAPFTest): + + def _CheckUnwrappedLines(self, uwlines, list_of_expected): + """Check that the given UnwrappedLines match expectations. + + Args: + uwlines: list of UnwrappedLine + list_of_expected: list of (depth, values) pairs. Non-semantic tokens are + filtered out from the expected values. + """ + actual = [] + for uwl in uwlines: + filtered_values = [ + ft.value + for ft in uwl.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS + ] + actual.append((uwl.depth, filtered_values)) + + self.assertEqual(list_of_expected, actual) + + def testSimpleFileScope(self): + code = textwrap.dedent(r""" + x = 1 + # a comment + y = 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['x', '=', '1']), + (0, ['# a comment']), + (0, ['y', '=', '2']), + ]) + + def testSimpleMultilineStatement(self): + code = textwrap.dedent(r""" + y = (1 + + x) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['y', '=', '(', '1', '+', 'x', ')']), + ]) + + def testFileScopeWithInlineComment(self): + code = textwrap.dedent(r""" + x = 1 # a comment + y = 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['x', '=', '1', '# a comment']), + (0, ['y', '=', '2']), + ]) + + def testSimpleIf(self): + code = textwrap.dedent(r""" + if foo: + x = 1 + y = 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['if', 'foo', ':']), + (1, ['x', '=', '1']), + (1, ['y', '=', '2']), + ]) + + def testSimpleIfWithComments(self): + code = textwrap.dedent(r""" + # c1 + if foo: # c2 + x = 1 + y = 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['# c1']), + (0, ['if', 'foo', ':', '# c2']), + (1, ['x', '=', '1']), + (1, ['y', '=', '2']), + ]) + + def testIfWithCommentsInside(self): + code = textwrap.dedent(r""" + if foo: + # c1 + x = 1 # c2 + # c3 + y = 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['if', 'foo', ':']), + (1, ['# c1']), + (1, ['x', '=', '1', '# c2']), + (1, ['# c3']), + (1, ['y', '=', '2']), + ]) + + def testIfElifElse(self): + code = textwrap.dedent(r""" + if x: + x = 1 # c1 + elif y: # c2 + y = 1 + else: + # c3 + z = 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['if', 'x', ':']), + (1, ['x', '=', '1', '# c1']), + (0, ['elif', 'y', ':', '# c2']), + (1, ['y', '=', '1']), + (0, ['else', ':']), + (1, ['# c3']), + (1, ['z', '=', '1']), + ]) + + def testNestedCompoundTwoLevel(self): + code = textwrap.dedent(r""" + if x: + x = 1 # c1 + while t: + # c2 + j = 1 + k = 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['if', 'x', ':']), + (1, ['x', '=', '1', '# c1']), + (1, ['while', 't', ':']), + (2, ['# c2']), + (2, ['j', '=', '1']), + (1, ['k', '=', '1']), + ]) + + def testSimpleWhile(self): + code = textwrap.dedent(r""" + while x > 1: # c1 + # c2 + x = 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['while', 'x', '>', '1', ':', '# c1']), + (1, ['# c2']), + (1, ['x', '=', '1']), + ]) + + def testSimpleTry(self): + code = textwrap.dedent(r""" + try: + pass + except: + pass + except: + pass + else: + pass + finally: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['try', ':']), + (1, ['pass']), + (0, ['except', ':']), + (1, ['pass']), + (0, ['except', ':']), + (1, ['pass']), + (0, ['else', ':']), + (1, ['pass']), + (0, ['finally', ':']), + (1, ['pass']), + ]) + + def testSimpleFuncdef(self): + code = textwrap.dedent(r""" + def foo(x): # c1 + # c2 + return x + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), + (1, ['# c2']), + (1, ['return', 'x']), + ]) + + def testTwoFuncDefs(self): + code = textwrap.dedent(r""" + def foo(x): # c1 + # c2 + return x + + def bar(): # c3 + # c4 + return x + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), + (1, ['# c2']), + (1, ['return', 'x']), + (0, ['def', 'bar', '(', ')', ':', '# c3']), + (1, ['# c4']), + (1, ['return', 'x']), + ]) + + def testSimpleClassDef(self): + code = textwrap.dedent(r""" + class Klass: # c1 + # c2 + p = 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['class', 'Klass', ':', '# c1']), + (1, ['# c2']), + (1, ['p', '=', '1']), + ]) + + def testSingleLineStmtInFunc(self): + code = textwrap.dedent(r""" + def f(): return 37 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['def', 'f', '(', ')', ':']), + (1, ['return', '37']), + ]) + + def testMultipleComments(self): + code = textwrap.dedent(r""" + # Comment #1 + + # Comment #2 + def f(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [ + (0, ['# Comment #1']), + (0, ['# Comment #2']), + (0, ['def', 'f', '(', ')', ':']), + (1, ['pass']), + ]) + + def testSplitListWithComment(self): + code = textwrap.dedent(r""" + a = [ + 'a', + 'b', + 'c', # hello world + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckUnwrappedLines(uwlines, [(0, [ + 'a', '=', '[', "'a'", ',', "'b'", ',', "'c'", ',', '# hello world', ']' + ])]) + + +class MatchBracketsTest(yapf_test_helper.YAPFTest): + + def _CheckMatchingBrackets(self, uwlines, list_of_expected): + """Check that the tokens have the expected matching bracket. + + Arguments: + uwlines: list of UnwrappedLine. + list_of_expected: list of (index, index) pairs. The matching brackets at + the indexes need to match. Non-semantic tokens are filtered out from the + expected values. + """ + actual = [] + for uwl in uwlines: + filtered_values = [(ft, ft.matching_bracket) + for ft in uwl.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] + if filtered_values: + actual.append(filtered_values) + + for index, bracket_list in enumerate(list_of_expected): + uwline = actual[index] + if not bracket_list: + for value in uwline: + self.assertIsNone(value[1]) + else: + for open_bracket, close_bracket in bracket_list: + self.assertEqual(uwline[open_bracket][0], uwline[close_bracket][1]) + self.assertEqual(uwline[close_bracket][0], uwline[open_bracket][1]) + + def testFunctionDef(self): + code = textwrap.dedent("""\ + def foo(a, b=['w','d'], c=[42, 37]): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(uwlines, [ + [(2, 20), (7, 11), (15, 19)], + [], + ]) + + def testDecorator(self): + code = textwrap.dedent("""\ + @bar() + def foo(a, b, c): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(uwlines, [ + [(2, 3)], + [(2, 8)], + [], + ]) + + def testClassDef(self): + code = textwrap.dedent("""\ + class A(B, C, D): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckMatchingBrackets(uwlines, [ + [(2, 8)], + [], + ]) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pytree_utils_test.py b/yapftests/pytree_utils_test.py new file mode 100644 index 0000000..3b9fde7 --- /dev/null +++ b/yapftests/pytree_utils_test.py @@ -0,0 +1,205 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.pytree_utils.""" + +import unittest + +from lib2to3 import pygram +from lib2to3 import pytree +from lib2to3.pgen2 import token + +from yapf.yapflib import pytree_utils + +# More direct access to the symbol->number mapping living within the grammar +# module. +_GRAMMAR_SYMBOL2NUMBER = pygram.python_grammar.symbol2number + +_FOO = 'foo' +_FOO1 = 'foo1' +_FOO2 = 'foo2' +_FOO3 = 'foo3' +_FOO4 = 'foo4' +_FOO5 = 'foo5' + + +class NodeNameTest(unittest.TestCase): + + def testNodeNameForLeaf(self): + leaf = pytree.Leaf(token.LPAR, '(') + self.assertEqual('LPAR', pytree_utils.NodeName(leaf)) + + def testNodeNameForNode(self): + leaf = pytree.Leaf(token.LPAR, '(') + node = pytree.Node(pygram.python_grammar.symbol2number['suite'], [leaf]) + self.assertEqual('suite', pytree_utils.NodeName(node)) + + +class ParseCodeToTreeTest(unittest.TestCase): + + def testParseCodeToTree(self): + # Since ParseCodeToTree is a thin wrapper around underlying lib2to3 + # functionality, only a sanity test here... + tree = pytree_utils.ParseCodeToTree('foo = 2\n') + self.assertEqual('file_input', pytree_utils.NodeName(tree)) + self.assertEqual(2, len(tree.children)) + self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) + + def testPrintFunctionToTree(self): + tree = pytree_utils.ParseCodeToTree( + 'print("hello world", file=sys.stderr)\n') + self.assertEqual('file_input', pytree_utils.NodeName(tree)) + self.assertEqual(2, len(tree.children)) + self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) + + def testPrintStatementToTree(self): + tree = pytree_utils.ParseCodeToTree('print "hello world"\n') + self.assertEqual('file_input', pytree_utils.NodeName(tree)) + self.assertEqual(2, len(tree.children)) + self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) + + def testClassNotLocal(self): + tree = pytree_utils.ParseCodeToTree('class nonlocal: pass\n') + self.assertEqual('file_input', pytree_utils.NodeName(tree)) + self.assertEqual(2, len(tree.children)) + self.assertEqual('classdef', pytree_utils.NodeName(tree.children[0])) + + +class InsertNodesBeforeAfterTest(unittest.TestCase): + + def _BuildSimpleTree(self): + # Builds a simple tree we can play with in the tests. + # The tree looks like this: + # + # suite: + # LPAR + # LPAR + # simple_stmt: + # NAME('foo') + # + lpar1 = pytree.Leaf(token.LPAR, '(') + lpar2 = pytree.Leaf(token.LPAR, '(') + simple_stmt = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'], + [pytree.Leaf(token.NAME, 'foo')]) + return pytree.Node(_GRAMMAR_SYMBOL2NUMBER['suite'], + [lpar1, lpar2, simple_stmt]) + + def _MakeNewNodeRPAR(self): + return pytree.Leaf(token.RPAR, ')') + + def setUp(self): + self._simple_tree = self._BuildSimpleTree() + + def testInsertNodesBefore(self): + # Insert before simple_stmt and make sure it went to the right place + pytree_utils.InsertNodesBefore([self._MakeNewNodeRPAR()], + self._simple_tree.children[2]) + self.assertEqual(4, len(self._simple_tree.children)) + self.assertEqual('RPAR', + pytree_utils.NodeName(self._simple_tree.children[2])) + self.assertEqual('simple_stmt', + pytree_utils.NodeName(self._simple_tree.children[3])) + + def testInsertNodesBeforeFirstChild(self): + # Insert before the first child of its parent + simple_stmt = self._simple_tree.children[2] + foo_child = simple_stmt.children[0] + pytree_utils.InsertNodesBefore([self._MakeNewNodeRPAR()], foo_child) + self.assertEqual(3, len(self._simple_tree.children)) + self.assertEqual(2, len(simple_stmt.children)) + self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[0])) + self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[1])) + + def testInsertNodesAfter(self): + # Insert after and make sure it went to the right place + pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()], + self._simple_tree.children[2]) + self.assertEqual(4, len(self._simple_tree.children)) + self.assertEqual('simple_stmt', + pytree_utils.NodeName(self._simple_tree.children[2])) + self.assertEqual('RPAR', + pytree_utils.NodeName(self._simple_tree.children[3])) + + def testInsertNodesAfterLastChild(self): + # Insert after the last child of its parent + simple_stmt = self._simple_tree.children[2] + foo_child = simple_stmt.children[0] + pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()], foo_child) + self.assertEqual(3, len(self._simple_tree.children)) + self.assertEqual(2, len(simple_stmt.children)) + self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[0])) + self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[1])) + + def testInsertNodesWhichHasParent(self): + # Try to insert an existing tree node into another place and fail. + with self.assertRaises(RuntimeError): + pytree_utils.InsertNodesAfter([self._simple_tree.children[1]], + self._simple_tree.children[0]) + + +class AnnotationsTest(unittest.TestCase): + + def setUp(self): + self._leaf = pytree.Leaf(token.LPAR, '(') + self._node = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'], + [pytree.Leaf(token.NAME, 'foo')]) + + def testGetWhenNone(self): + self.assertIsNone(pytree_utils.GetNodeAnnotation(self._leaf, _FOO)) + + def testSetWhenNone(self): + pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) + + def testSetAgain(self): + pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 30) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 30) + + def testMultiple(self): + pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO1, 1) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO2, 2) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO3, 3) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO4, 4) + pytree_utils.SetNodeAnnotation(self._leaf, _FOO5, 5) + + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO1), 1) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO2), 2) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO3), 3) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO4), 4) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO5), 5) + + def testSubtype(self): + pytree_utils.AppendNodeAnnotation(self._leaf, + pytree_utils.Annotation.SUBTYPE, _FOO) + + self.assertSetEqual( + pytree_utils.GetNodeAnnotation(self._leaf, + pytree_utils.Annotation.SUBTYPE), {_FOO}) + + pytree_utils.RemoveSubtypeAnnotation(self._leaf, _FOO) + + self.assertSetEqual( + pytree_utils.GetNodeAnnotation(self._leaf, + pytree_utils.Annotation.SUBTYPE), set()) + + def testSetOnNode(self): + pytree_utils.SetNodeAnnotation(self._node, _FOO, 20) + self.assertEqual(pytree_utils.GetNodeAnnotation(self._node, _FOO), 20) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/pytree_visitor_test.py b/yapftests/pytree_visitor_test.py new file mode 100644 index 0000000..1908249 --- /dev/null +++ b/yapftests/pytree_visitor_test.py @@ -0,0 +1,120 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.pytree_visitor.""" + +import unittest + +from yapf.yapflib import py3compat +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor + + +class _NodeNameCollector(pytree_visitor.PyTreeVisitor): + """A tree visitor that collects the names of all tree nodes into a list. + + Attributes: + all_node_names: collected list of the names, available when the traversal + is over. + name_node_values: collects a list of NAME leaves (in addition to those going + into all_node_names). + """ + + def __init__(self): + self.all_node_names = [] + self.name_node_values = [] + + def DefaultNodeVisit(self, node): + self.all_node_names.append(pytree_utils.NodeName(node)) + super(_NodeNameCollector, self).DefaultNodeVisit(node) + + def DefaultLeafVisit(self, leaf): + self.all_node_names.append(pytree_utils.NodeName(leaf)) + + def Visit_NAME(self, leaf): + self.name_node_values.append(leaf.value) + self.DefaultLeafVisit(leaf) + + +_VISITOR_TEST_SIMPLE_CODE = r""" +foo = bar +baz = x +""" + +_VISITOR_TEST_NESTED_CODE = r""" +if x: + if y: + return z +""" + + +class PytreeVisitorTest(unittest.TestCase): + + def testCollectAllNodeNamesSimpleCode(self): + tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) + collector = _NodeNameCollector() + collector.Visit(tree) + expected_names = [ + 'file_input', + 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', + 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', + 'ENDMARKER', + ] # yapf: disable + self.assertEqual(expected_names, collector.all_node_names) + + expected_name_node_values = ['foo', 'bar', 'baz', 'x'] + self.assertEqual(expected_name_node_values, collector.name_node_values) + + def testCollectAllNodeNamesNestedCode(self): + tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_NESTED_CODE) + collector = _NodeNameCollector() + collector.Visit(tree) + expected_names = [ + 'file_input', + 'if_stmt', 'NAME', 'NAME', 'COLON', + 'suite', 'NEWLINE', + 'INDENT', 'if_stmt', 'NAME', 'NAME', 'COLON', 'suite', 'NEWLINE', + 'INDENT', 'simple_stmt', 'return_stmt', 'NAME', 'NAME', 'NEWLINE', + 'DEDENT', 'DEDENT', 'ENDMARKER', + ] # yapf: disable + self.assertEqual(expected_names, collector.all_node_names) + + expected_name_node_values = ['if', 'x', 'if', 'y', 'return', 'z'] + self.assertEqual(expected_name_node_values, collector.name_node_values) + + def testDumper(self): + # PyTreeDumper is mainly a debugging utility, so only do basic sanity + # checking. + tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) + stream = py3compat.StringIO() + pytree_visitor.PyTreeDumper(target_stream=stream).Visit(tree) + + dump_output = stream.getvalue() + self.assertIn('file_input [3 children]', dump_output) + self.assertIn("NAME(Leaf(NAME, 'foo'))", dump_output) + self.assertIn("EQUAL(Leaf(EQUAL, '='))", dump_output) + + def testDumpPyTree(self): + # Similar sanity checking for the convenience wrapper DumpPyTree + tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) + stream = py3compat.StringIO() + pytree_visitor.DumpPyTree(tree, target_stream=stream) + + dump_output = stream.getvalue() + self.assertIn('file_input [3 children]', dump_output) + self.assertIn("NAME(Leaf(NAME, 'foo'))", dump_output) + self.assertIn("EQUAL(Leaf(EQUAL, '='))", dump_output) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_basic_test.py b/yapftests/reformatter_basic_test.py new file mode 100644 index 0000000..a3de63d --- /dev/null +++ b/yapftests/reformatter_basic_test.py @@ -0,0 +1,2509 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Basic tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class BasicReformatterTest(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testSplittingAllArgs(self): + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{split_all_comma_separated_values: true, column_limit: 40}')) + unformatted_code = textwrap.dedent("""\ + responseDict = {"timestamp": timestamp, "someValue": value, "whatever": 120} + """) + expected_formatted_code = textwrap.dedent("""\ + responseDict = { + "timestamp": timestamp, + "someValue": value, + "whatever": 120 + } + """) + unformatted_code = textwrap.dedent("""\ + def foo(long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + unformatted_code = textwrap.dedent("""\ + foo_tuple = [long_arg, really_long_arg, really_really_long_arg, cant_keep_all_these_args] + """) + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [ + long_arg, + really_long_arg, + really_really_long_arg, + cant_keep_all_these_args + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + unformatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + expected_formatted_code = textwrap.dedent("""\ + foo_tuple = [short, arg] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSimpleFunctionsWithTrailingComments(self): + unformatted_code = textwrap.dedent("""\ + def g(): # Trailing comment + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def f( # Intermediate comment + ): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def g(): # Trailing comment + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def f( # Intermediate comment + ): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBlankLinesAtEndOfFile(self): + unformatted_code = textwrap.dedent("""\ + def foobar(): # foo + pass + + + + """) + expected_formatted_code = textwrap.dedent("""\ + def foobar(): # foo + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + x = { 'a':37,'b':42, + + 'c':927} + + """) + expected_formatted_code = textwrap.dedent("""\ + x = {'a': 37, 'b': 42, 'c': 927} + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMultipleUgliness(self): + unformatted_code = textwrap.dedent("""\ + x = { 'a':37,'b':42, + + 'c':927} + + y = 'hello ''world' + z = 'hello '+'world' + a = 'hello {}'.format('world') + class foo ( object ): + def f (self ): + return 37*-+2 + def g(self, x,y=42): + return y + def f ( a ) : + return 37+-+a[42-x : y**3] + """) + expected_formatted_code = textwrap.dedent("""\ + x = {'a': 37, 'b': 42, 'c': 927} + + y = 'hello ' 'world' + z = 'hello ' + 'world' + a = 'hello {}'.format('world') + + + class foo(object): + + def f(self): + return 37 * -+2 + + def g(self, x, y=42): + return y + + + def f(a): + return 37 + -+a[42 - x:y**3] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testComments(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + pass + + # Attached comment + class Bar(object): + pass + + global_assignment = 42 + + # Comment attached to class with decorator. + # Comment attached to class with decorator. + @noop + @noop + class Baz(object): + pass + + # Intermediate comment + + class Qux(object): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + pass + + + # Attached comment + class Bar(object): + pass + + + global_assignment = 42 + + + # Comment attached to class with decorator. + # Comment attached to class with decorator. + @noop + @noop + class Baz(object): + pass + + + # Intermediate comment + + + class Qux(object): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSingleComment(self): + code = textwrap.dedent("""\ + # Thing 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentsWithTrailingSpaces(self): + unformatted_code = textwrap.dedent("""\ + # Thing 1 + # Thing 2 + """) + expected_formatted_code = textwrap.dedent("""\ + # Thing 1 + # Thing 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testCommentsInDataLiteral(self): + code = textwrap.dedent("""\ + def f(): + return collections.OrderedDict({ + # First comment. + 'fnord': 37, + + # Second comment. + # Continuation of second comment. + 'bork': 42, + + # Ending comment. + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testEndingWhitespaceAfterSimpleStatement(self): + code = textwrap.dedent("""\ + import foo as bar + # Thing 1 + # Thing 2 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDocstrings(self): + unformatted_code = textwrap.dedent('''\ + u"""Module-level docstring.""" + import os + class Foo(object): + + """Class-level docstring.""" + # A comment for qux. + def qux(self): + + + """Function-level docstring. + + A multiline function docstring. + """ + print('hello {}'.format('world')) + return 42 + ''') + expected_formatted_code = textwrap.dedent('''\ + u"""Module-level docstring.""" + import os + + + class Foo(object): + """Class-level docstring.""" + + # A comment for qux. + def qux(self): + """Function-level docstring. + + A multiline function docstring. + """ + print('hello {}'.format('world')) + return 42 + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDocstringAndMultilineComment(self): + unformatted_code = textwrap.dedent('''\ + """Hello world""" + # A multiline + # comment + class bar(object): + """class docstring""" + # class multiline + # comment + def foo(self): + """Another docstring.""" + # Another multiline + # comment + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + """Hello world""" + + + # A multiline + # comment + class bar(object): + """class docstring""" + + # class multiline + # comment + def foo(self): + """Another docstring.""" + # Another multiline + # comment + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMultilineDocstringAndMultilineComment(self): + unformatted_code = textwrap.dedent('''\ + """Hello world + + RIP Dennis Richie. + """ + # A multiline + # comment + class bar(object): + """class docstring + + A classy class. + """ + # class multiline + # comment + def foo(self): + """Another docstring. + + A functional function. + """ + # Another multiline + # comment + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + """Hello world + + RIP Dennis Richie. + """ + + + # A multiline + # comment + class bar(object): + """class docstring + + A classy class. + """ + + # class multiline + # comment + def foo(self): + """Another docstring. + + A functional function. + """ + # Another multiline + # comment + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testTupleCommaBeforeLastParen(self): + unformatted_code = textwrap.dedent("""\ + a = ( 1, ) + """) + expected_formatted_code = textwrap.dedent("""\ + a = (1,) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoBreakOutsideOfBracket(self): + # FIXME(morbo): How this is formatted is not correct. But it's syntactically + # correct. + unformatted_code = textwrap.dedent("""\ + def f(): + assert port >= minimum, \ +'Unexpected port %d when minimum was %d.' % (port, minimum) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + assert port >= minimum, 'Unexpected port %d when minimum was %d.' % (port, + minimum) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBlankLinesBeforeDecorators(self): + unformatted_code = textwrap.dedent("""\ + @foo() + class A(object): + @bar() + @baz() + def x(self): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + @foo() + class A(object): + + @bar() + @baz() + def x(self): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testCommentBetweenDecorators(self): + unformatted_code = textwrap.dedent("""\ + @foo() + # frob + @bar + def x (self): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + @foo() + # frob + @bar + def x(self): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testListComprehension(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + [k for k in () + if k in y] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + [k for k in () if k in y] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testListComprehensionPreferOneLine(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name in () + if long_var_name == 2] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 for long_var_name in () if long_var_name == 2 + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testListComprehensionPreferOneLineOverArithmeticSplit(self): + unformatted_code = textwrap.dedent("""\ + def given(used_identifiers): + return (sum(len(identifier) + for identifier in used_identifiers) / len(used_identifiers)) + """) + expected_formatted_code = textwrap.dedent("""\ + def given(used_identifiers): + return (sum(len(identifier) for identifier in used_identifiers) / + len(used_identifiers)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testListComprehensionPreferThreeLinesForLineWrap(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + 1 + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3 + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testListComprehensionPreferNoBreakForTrivialExpression(self): + unformatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name + for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3] + """) + expected_formatted_code = textwrap.dedent("""\ + def given(y): + long_variable_name = [ + long_var_name for long_var_name, number_two in () + if long_var_name == 2 and number_two == 3 + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testOpeningAndClosingBrackets(self): + unformatted_code = """\ +foo( (1, ) ) +foo( ( 1, 2, 3 ) ) +foo( ( 1, 2, 3, ) ) +""" + expected_formatted_code = """\ +foo((1,)) +foo((1, 2, 3)) +foo(( + 1, + 2, + 3, +)) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSingleLineFunctions(self): + unformatted_code = textwrap.dedent("""\ + def foo(): return 42 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return 42 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoQueueSeletionInMiddleOfLine(self): + # If the queue isn't properly constructed, then a token in the middle of the + # line may be selected as the one with least penalty. The tokens after that + # one are then splatted at the end of the line with no formatting. + unformatted_code = """\ +find_symbol(node.type) + "< " + " ".join(find_pattern(n) for n in node.child) + " >" +""" + expected_formatted_code = """\ +find_symbol(node.type) + "< " + " ".join( + find_pattern(n) for n in node.child) + " >" +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoSpacesBetweenSubscriptsAndCalls(self): + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc() [42] (a, 2) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc()[42](a, 2) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoSpacesBetweenOpeningBracketAndStartingOperator(self): + # Unary operator. + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc[ -1 ]( -42 ) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc[-1](-42) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + # Varargs and kwargs. + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc( *varargs ) + aaaaaaaaaa = bbbbbbbb.ccccccccc( **kwargs ) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaa = bbbbbbbb.ccccccccc(*varargs) + aaaaaaaaaa = bbbbbbbb.ccccccccc(**kwargs) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMultilineCommentReformatted(self): + unformatted_code = textwrap.dedent("""\ + if True: + # This is a multiline + # comment. + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + # This is a multiline + # comment. + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDictionaryMakerFormatting(self): + unformatted_code = textwrap.dedent("""\ + _PYTHON_STATEMENTS = frozenset({ + lambda x, y: 'simple_stmt': 'small_stmt', 'expr_stmt': 'print_stmt', 'del_stmt': + 'pass_stmt', lambda: 'break_stmt': 'continue_stmt', 'return_stmt': 'raise_stmt', + 'yield_stmt': 'import_stmt', lambda: 'global_stmt': 'exec_stmt', 'assert_stmt': + 'if_stmt', 'while_stmt': 'for_stmt', + }) + """) + expected_formatted_code = textwrap.dedent("""\ + _PYTHON_STATEMENTS = frozenset({ + lambda x, y: 'simple_stmt': 'small_stmt', + 'expr_stmt': 'print_stmt', + 'del_stmt': 'pass_stmt', + lambda: 'break_stmt': 'continue_stmt', + 'return_stmt': 'raise_stmt', + 'yield_stmt': 'import_stmt', + lambda: 'global_stmt': 'exec_stmt', + 'assert_stmt': 'if_stmt', + 'while_stmt': 'for_stmt', + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSimpleMultilineCode(self): + unformatted_code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ +xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, \ +xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, + vvvvvvvvv) + aaaaaaaaaaaaaa.bbbbbbbbbbbbbb.ccccccc(zzzzzzzzzzzz, xxxxxxxxxxx, yyyyyyyyyyyy, + vvvvvvvvv) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMultilineComment(self): + code = textwrap.dedent("""\ + if Foo: + # Hello world + # Yo man. + # Yo man. + # Yo man. + # Yo man. + a = 42 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testMultilineString(self): + code = textwrap.dedent("""\ + code = textwrap.dedent('''\ + if Foo: + # Hello world + # Yo man. + # Yo man. + # Yo man. + # Yo man. + a = 42 + ''') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent('''\ + def f(): + email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br> + <b>Czar: </b>"""+despot["Nicholas"]+"""<br> + <b>Minion: </b>"""+serf["Dmitri"]+"""<br> + <b>Residence: </b>"""+palace["Winter"]+"""<br> + </body> + </html>""" + ''') + expected_formatted_code = textwrap.dedent('''\ + def f(): + email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br> + <b>Czar: </b>""" + despot["Nicholas"] + """<br> + <b>Minion: </b>""" + serf["Dmitri"] + """<br> + <b>Residence: </b>""" + palace["Winter"] + """<br> + </body> + </html>""" + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSimpleMultilineWithComments(self): + code = textwrap.dedent("""\ + if ( # This is the first comment + a and # This is the second comment + # This is the third comment + b): # A trailing comment + # Whoa! A normal comment!! + pass # Another trailing comment + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testMatchingParenSplittingMatching(self): + unformatted_code = textwrap.dedent("""\ + def f(): + raise RuntimeError('unable to find insertion point for target node', + (target,)) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + raise RuntimeError('unable to find insertion point for target node', + (target,)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testContinuationIndent(self): + unformatted_code = textwrap.dedent('''\ + class F: + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + if isinstance(child, pytree.Leaf): + self._SetTokenSubtype( + child, subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get( + child.value, format_token.Subtype.NONE)) + ''') + expected_formatted_code = textwrap.dedent('''\ + class F: + + def _ProcessArgLists(self, node): + """Common method for processing argument lists.""" + for child in node.children: + if isinstance(child, pytree.Leaf): + self._SetTokenSubtype( + child, + subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, + format_token.Subtype.NONE)) + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testTrailingCommaAndBracket(self): + unformatted_code = textwrap.dedent('''\ + a = { 42, } + b = ( 42, ) + c = [ 42, ] + ''') + expected_formatted_code = textwrap.dedent('''\ + a = { + 42, + } + b = (42,) + c = [ + 42, + ] + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testI18n(self): + code = textwrap.dedent("""\ + N_('Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world.') # A comment is here. + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + foo('Fake function call') #. Some years ago - never mind how long precisely - having little or no money in my purse, and nothing particular to interest me on shore, I thought I would sail about a little and see the watery part of the world. + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testI18nCommentsInDataLiteral(self): + code = textwrap.dedent("""\ + def f(): + return collections.OrderedDict({ + #. First i18n comment. + 'bork': 'foo', + + #. Second i18n comment. + 'snork': 'bar#.*=\\\\0', + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testClosingBracketIndent(self): + code = textwrap.dedent('''\ + def f(): + + def g(): + while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxxx( + yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testClosingBracketsInlinedInCall(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( + self.cccccc.ddddddddd.eeeeeeee, + options={ + "forkforkfork": 1, + "borkborkbork": 2, + "corkcorkcork": 3, + "horkhorkhork": 4, + "porkporkpork": 5, + }) + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + self.aaaaaaaa = xxxxxxxxxxxxxxxxxxx.yyyyyyyyyyyyy( + self.cccccc.ddddddddd.eeeeeeee, + options={ + "forkforkfork": 1, + "borkborkbork": 2, + "corkcorkcork": 3, + "horkhorkhork": 4, + "porkporkpork": 5, + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testLineWrapInForExpression(self): + code = textwrap.dedent("""\ + class A: + + def x(self, node, name, n=1): + for i, child in enumerate( + itertools.ifilter(lambda c: pytree_utils.NodeName(c) == name, + node.pre_order())): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testFunctionCallContinuationLine(self): + code = """\ +class foo: + + def bar(self, node, name, n=1): + if True: + if True: + return [(aaaaaaaaaa, + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb( + cccc, ddddddddddddddddddddddddddddddddddddd))] +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testI18nNonFormatting(self): + code = textwrap.dedent("""\ + class F(object): + + def __init__(self, fieldname, + #. Error message indicating an invalid e-mail address. + message=N_('Please check your email address.'), **kwargs): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSpaceBetweenUnaryOpAndOpeningParen(self): + code = textwrap.dedent("""\ + if ~(a or b): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentBeforeFuncDef(self): + code = textwrap.dedent("""\ + class Foo(object): + + a = 42 + + # This is a comment. + def __init__(self, + xxxxxxx, + yyyyy=0, + zzzzzzz=None, + aaaaaaaaaaaaaaaaaa=False, + bbbbbbbbbbbbbbb=False): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testExcessLineCountWithDefaultKeywords(self): + unformatted_code = textwrap.dedent("""\ + class Fnord(object): + def Moo(self): + aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( + ccccccccccccc=ccccccccccccc, ddddddd=ddddddd, eeee=eeee, + fffff=fffff, ggggggg=ggggggg, hhhhhhhhhhhhh=hhhhhhhhhhhhh, + iiiiiii=iiiiiiiiiiiiii) + """) + expected_formatted_code = textwrap.dedent("""\ + class Fnord(object): + + def Moo(self): + aaaaaaaaaaaaaaaa = self._bbbbbbbbbbbbbbbbbbbbbbb( + ccccccccccccc=ccccccccccccc, + ddddddd=ddddddd, + eeee=eeee, + fffff=fffff, + ggggggg=ggggggg, + hhhhhhhhhhhhh=hhhhhhhhhhhhh, + iiiiiii=iiiiiiiiiiiiii) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSpaceAfterNotOperator(self): + code = textwrap.dedent("""\ + if not (this and that): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoPenaltySplitting(self): + code = textwrap.dedent("""\ + def f(): + if True: + if True: + python_files.extend( + os.path.join(filename, f) + for f in os.listdir(filename) + if IsPythonFile(os.path.join(filename, f))) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testExpressionPenalties(self): + code = textwrap.dedent("""\ + def f(): + if ((left.value == '(' and right.value == ')') or + (left.value == '[' and right.value == ']') or + (left.value == '{' and right.value == '}')): + return False + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testLineDepthOfSingleLineStatement(self): + unformatted_code = textwrap.dedent("""\ + while True: continue + for x in range(3): continue + try: a = 42 + except: b = 42 + with open(a) as fd: a = fd.read() + """) + expected_formatted_code = textwrap.dedent("""\ + while True: + continue + for x in range(3): + continue + try: + a = 42 + except: + b = 42 + with open(a) as fd: + a = fd.read() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplitListWithTerminatingComma(self): + unformatted_code = textwrap.dedent("""\ + FOO = ['bar', 'baz', 'mux', 'qux', 'quux', 'quuux', 'quuuux', + 'quuuuux', 'quuuuuux', 'quuuuuuux', lambda a, b: 37,] + """) + expected_formatted_code = textwrap.dedent("""\ + FOO = [ + 'bar', + 'baz', + 'mux', + 'qux', + 'quux', + 'quuux', + 'quuuux', + 'quuuuux', + 'quuuuuux', + 'quuuuuuux', + lambda a, b: 37, + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplitListWithInterspersedComments(self): + code = textwrap.dedent("""\ + FOO = [ + 'bar', # bar + 'baz', # baz + 'mux', # mux + 'qux', # qux + 'quux', # quux + 'quuux', # quuux + 'quuuux', # quuuux + 'quuuuux', # quuuuux + 'quuuuuux', # quuuuuux + 'quuuuuuux', # quuuuuuux + lambda a, b: 37 # lambda + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testRelativeImportStatements(self): + code = textwrap.dedent("""\ + from ... import bork + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testSingleLineList(self): + # A list on a single line should prefer to remain contiguous. + unformatted_code = textwrap.dedent("""\ + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( + ("...", "."), "..", + ".............................................." + ) + """) + expected_formatted_code = textwrap.dedent("""\ + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = aaaaaaaaaaa( + ("...", "."), "..", "..............................................") + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBlankLinesBeforeFunctionsNotInColumnZero(self): + unformatted_code = textwrap.dedent("""\ + import signal + + + try: + signal.SIGALRM + # .................................................................. + # ............................................................... + + + def timeout(seconds=1): + pass + except: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import signal + + try: + signal.SIGALRM + + # .................................................................. + # ............................................................... + + + def timeout(seconds=1): + pass + except: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoKeywordArgumentBreakage(self): + code = textwrap.dedent("""\ + class A(object): + + def b(self): + if self.aaaaaaaaaaaaaaaaaaaa not in self.bbbbbbbbbb( + cccccccccccccccccccc=True): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testTrailerOnSingleLine(self): + code = """\ +urlpatterns = patterns('', url(r'^$', 'homepage_view'), + url(r'^/login/$', 'login_view'), + url(r'^/login/$', 'logout_view'), + url(r'^/user/(?P<username>\\w+)/$', 'profile_view')) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testIfConditionalParens(self): + code = textwrap.dedent("""\ + class Foo: + + def bar(): + if True: + if (child.type == grammar_token.NAME and + child.value in substatement_names): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testContinuationMarkers(self): + code = textwrap.dedent("""\ + text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec a diam lectus. "\\ + "Sed sit amet ipsum mauris. Maecenas congue ligula ac quam viverra nec consectetur "\\ + "ante hendrerit. Donec et mollis dolor. Praesent et diam eget libero egestas mattis "\\ + "sit amet vitae augue. Nam tincidunt congue enim, ut porta lorem lacinia consectetur. "\\ + "Donec ut libero sed arcu vehicula ultricies a non tortor. Lorem ipsum dolor sit amet" + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + from __future__ import nested_scopes, generators, division, absolute_import, with_statement, \\ + print_function, unicode_literals + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + if aaaaaaaaa == 42 and bbbbbbbbbbbbbb == 42 and \\ + cccccccc == 42: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentsWithContinuationMarkers(self): + code = textwrap.dedent("""\ + def fn(arg): + v = fn2(key1=True, + #c1 + key2=arg)\\ + .fn3() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testMultipleContinuationMarkers(self): + code = textwrap.dedent("""\ + xyz = \\ + \\ + some_thing() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testContinuationMarkerAfterStringWithContinuation(self): + code = """\ +s = 'foo \\ + bar' \\ + .format() +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testEmptyContainers(self): + code = textwrap.dedent("""\ + flags.DEFINE_list( + 'output_dirs', [], + 'Lorem ipsum dolor sit amet, consetetur adipiscing elit. Donec a diam lectus. ' + 'Sed sit amet ipsum mauris. Maecenas congue.') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testSplitStringsIfSurroundedByParens(self): + unformatted_code = textwrap.dedent("""\ + a = foo.bar({'xxxxxxxxxxxxxxxxxxxxxxx' 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42]} + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' 'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' 'ddddddddddddddddddddddddddddd') + """) + expected_formatted_code = textwrap.dedent("""\ + a = foo.bar({ + 'xxxxxxxxxxxxxxxxxxxxxxx' + 'yyyyyyyyyyyyyyyyyyyyyyyyyy': baz[42] + } + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'bbbbbbbbbbbbbbbbbbbbbbbbbb' + 'cccccccccccccccccccccccccccccccc' + 'ddddddddddddddddddddddddddddd') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + a = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \ +'bbbbbbbbbbbbbbbbbbbbbbbbbb' 'cccccccccccccccccccccccccccccccc' \ +'ddddddddddddddddddddddddddddd' + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testMultilineShebang(self): + code = textwrap.dedent("""\ + #!/bin/sh + if "true" : '''\' + then + + export FOO=123 + exec /usr/bin/env python "$0" "$@" + + exit 127 + fi + ''' + + import os + + assert os.environ['FOO'] == '123' + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSplittingAroundTermOperators(self): + code = textwrap.dedent("""\ + a_very_long_function_call_yada_yada_etc_etc_etc(long_arg1, + long_arg2 / long_arg3) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSplittingWithinSubscriptList(self): + code = textwrap.dedent("""\ + somequitelongvariablename.somemember[(a, b)] = { + 'somelongkey': 1, + 'someotherlongkey': 2 + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testExcessCharacters(self): + code = textwrap.dedent("""\ + class foo: + + def bar(self): + self.write(s=[ + '%s%s %s' % ('many of really', 'long strings', '+ just makes up 81') + ]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if contract == allow_contract and attr_dict.get(if_attribute) == has_value: + return True + """) + expected_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if contract == allow_contract and attr_dict.get( + if_attribute) == has_value: + return True + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testDictSetGenerator(self): + code = textwrap.dedent("""\ + foo = { + variable: 'hello world. How are you today?' + for variable in fnord + if variable != 37 + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testUnaryOpInDictionaryValue(self): + code = textwrap.dedent("""\ + beta = "123" + + test = {'alpha': beta[-1]} + + print(beta[-1]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testUnaryNotOperator(self): + code = textwrap.dedent("""\ + if True: + if True: + if True: + if True: + remote_checksum = self.get_checksum(conn, tmp, dest, inject, + not directory_prepended, source) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testRelaxArraySubscriptAffinity(self): + code = textwrap.dedent("""\ + class A(object): + + def f(self, aaaaaaaaa, bbbbbbbbbbbbb, row): + if True: + if True: + if True: + if True: + if row[4] is None or row[5] is None: + bbbbbbbbbbbbb['..............'] = row[ + 5] if row[5] is not None else 5 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testFunctionCallInDict(self): + code = "a = {'a': b(c=d, **e)}\n" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testFunctionCallInNestedDict(self): + code = "a = {'a': {'a': {'a': b(c=d, **e)}}}\n" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testUnbreakableNot(self): + code = textwrap.dedent("""\ + def test(): + if not "Foooooooooooooooooooooooooooooo" or "Foooooooooooooooooooooooooooooo" == "Foooooooooooooooooooooooooooooo": + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testSplitListWithComment(self): + code = textwrap.dedent("""\ + a = [ + 'a', + 'b', + 'c' # hello world + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testOverColumnLimit(self): + unformatted_code = textwrap.dedent("""\ + class Test: + + def testSomething(self): + expected = { + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): 'ccccccccccccccccccccccccccccccccccccccccccc', + } + """) + expected_formatted_code = textwrap.dedent("""\ + class Test: + + def testSomething(self): + expected = { + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + ('aaaaaaaaaaaaa', 'bbbb'): + 'ccccccccccccccccccccccccccccccccccccccccccc', + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testEndingComment(self): + code = textwrap.dedent("""\ + a = f( + a="something", + b="something requiring comment which is quite long", # comment about b (pushes line over 79) + c="something else, about which comment doesn't make sense") + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testContinuationSpaceRetention(self): + code = textwrap.dedent("""\ + def fn(): + return module \\ + .method(Object(data, + fn2(arg) + )) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testIfExpressionWithFunctionCall(self): + code = textwrap.dedent("""\ + if x or z.y( + a, + c, + aaaaaaaaaaaaaaaaaaaaa=aaaaaaaaaaaaaaaaaa, + bbbbbbbbbbbbbbbbbbbbb=bbbbbbbbbbbbbbbbbb): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testUnformattedAfterMultilineString(self): + code = textwrap.dedent("""\ + def foo(): + com_text = \\ + ''' + TEST + ''' % (input_fname, output_fname) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSpacesAroundKeywordDefaultValues(self): + code = textwrap.dedent("""\ + sources = { + 'json': request.get_json(silent=True) or {}, + 'json2': request.get_json(silent=True), + } + json = request.get_json(silent=True) or {} + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSplittingBeforeEndingSubscriptBracket(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + status = cf.describe_stacks(StackName=stackname)[u'Stacks'][0][u'StackStatus'] + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + status = cf.describe_stacks( + StackName=stackname)[u'Stacks'][0][u'StackStatus'] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoSplittingOnSingleArgument(self): + unformatted_code = textwrap.dedent("""\ + xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', + aaaaaaa.bbbbbbbbbbbb).group(1) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', + ccccccc).group(1)) + xxxxxxxxxxxxxx = (re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', + aaaaaaa.bbbbbbbbbbbb).group(a.b) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', + ccccccc).group(c.d)) + """) + expected_formatted_code = textwrap.dedent("""\ + xxxxxxxxxxxxxx = ( + re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(1) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(1)) + xxxxxxxxxxxxxx = ( + re.search(r'(\\d+\\.\\d+\\.\\d+\\.)\\d+', aaaaaaa.bbbbbbbbbbbb).group(a.b) + + re.search(r'\\d+\\.\\d+\\.\\d+\\.(\\d+)', ccccccc).group(c.d)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingArraysSensibly(self): + unformatted_code = textwrap.dedent("""\ + while True: + while True: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list['bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list('bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') + """) + expected_formatted_code = textwrap.dedent("""\ + while True: + while True: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list[ + 'bbbbbbbbbbbbbbbbbbbbbbbbb'].split(',') + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = list( + 'bbbbbbbbbbbbbbbbbbbbbbbbb').split(',') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testComprehensionForAndIf(self): + unformatted_code = textwrap.dedent("""\ + class f: + + def __repr__(self): + tokens_repr = ','.join(['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) + """) + expected_formatted_code = textwrap.dedent("""\ + class f: + + def __repr__(self): + tokens_repr = ','.join( + ['{0}({1!r})'.format(tok.name, tok.value) for tok in self._tokens]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testFunctionCallArguments(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, + standalone=True), ancestor_at_indent) + pytree_utils.InsertNodesBefore(_CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, + standalone=True)) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + pytree_utils.InsertNodesBefore( + _CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, standalone=True), + ancestor_at_indent) + pytree_utils.InsertNodesBefore( + _CreateCommentsFromPrefix( + comment_prefix, comment_lineno, comment_column, standalone=True)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBinaryOperators(self): + unformatted_code = textwrap.dedent("""\ + a = b ** 37 + c = (20 ** -3) / (_GRID_ROWS ** (code_length - 10)) + """) + expected_formatted_code = textwrap.dedent("""\ + a = b**37 + c = (20**-3) / (_GRID_ROWS**(code_length - 10)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + def f(): + if True: + if (self.stack[-1].split_before_closing_bracket and + # FIXME(morbo): Use the 'matching_bracket' instead of this. + # FIXME(morbo): Don't forget about tuples! + current.value in ']}'): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testContiguousList(self): + code = textwrap.dedent("""\ + [retval1, retval2] = a_very_long_function(argument_1, argument2, argument_3, + argument_4) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testArgsAndKwargsFormatting(self): + code = textwrap.dedent("""\ + a(a=aaaaaaaaaaaaaaaaaaaaa, + b=aaaaaaaaaaaaaaaaaaaaaaaa, + c=aaaaaaaaaaaaaaaaaa, + *d, + **e) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + def foo(): + return [ + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testCommentColumnLimitOverflow(self): + code = textwrap.dedent("""\ + def f(): + if True: + TaskManager.get_tags = MagicMock( + name='get_tags_mock', + return_value=[157031694470475], + # side_effect=[(157031694470475), (157031694470475),], + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testMultilineLambdas(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, allow_multiline_lambdas: true}')) + unformatted_code = textwrap.dedent("""\ + class SomeClass(object): + do_something = True + + def succeeded(self, dddddddddddddd): + d = defer.succeed(None) + + if self.do_something: + d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccc(dddddddddddddd)) + return d + """) + expected_formatted_code = textwrap.dedent("""\ + class SomeClass(object): + do_something = True + + def succeeded(self, dddddddddddddd): + d = defer.succeed(None) + + if self.do_something: + d.addCallback(lambda _: self.aaaaaa.bbbbbbbbbbbbbbbb. + cccccccccccccccccccccccccccccccc(dddddddddddddd)) + return d + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testMultilineDictionaryKeys(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: chromium, ' + 'allow_multiline_dictionary_keys: true}')) + unformatted_code = textwrap.dedent("""\ + MAP_WITH_LONG_KEYS = { + ('lorem ipsum', 'dolor sit amet'): + 1, + ('consectetur adipiscing elit.', 'Vestibulum mauris justo, ornare eget dolor eget'): + 2, + ('vehicula convallis nulla. Vestibulum dictum nisl in malesuada finibus.',): + 3 + } + """) + expected_formatted_code = textwrap.dedent("""\ + MAP_WITH_LONG_KEYS = { + ('lorem ipsum', 'dolor sit amet'): + 1, + ('consectetur adipiscing elit.', + 'Vestibulum mauris justo, ornare eget dolor eget'): + 2, + ('vehicula convallis nulla. Vestibulum dictum nisl in malesuada finibus.',): + 3 + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testStableDictionaryFormatting(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, indent_width: 2, ' + 'continuation_indent_width: 4, indent_dictionary_value: True}')) + code = textwrap.dedent("""\ + class A(object): + def method(self): + filters = { + 'expressions': [{ + 'field': { + 'search_field': { + 'user_field': 'latest_party__number_of_guests' + }, + } + }] + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + + uwlines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testStableInlinedDictionaryFormatting(self): + try: + style.SetGlobalStyle(style.CreatePEP8Style()) + unformatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({'action': 'update', 'parameter': value})) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + url = "http://{0}/axis-cgi/admin/param.cgi?{1}".format( + value, urllib.urlencode({ + 'action': 'update', + 'parameter': value + })) + """) + + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + uwlines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testDontSplitKeywordValueArguments(self): + unformatted_code = textwrap.dedent("""\ + def mark_game_scored(gid): + _connect.execute(_games.update().where(_games.c.gid == gid).values( + scored=True)) + """) + expected_formatted_code = textwrap.dedent("""\ + def mark_game_scored(gid): + _connect.execute( + _games.update().where(_games.c.gid == gid).values(scored=True)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDontAddBlankLineAfterMultilineString(self): + code = textwrap.dedent("""\ + query = '''SELECT id + FROM table + WHERE day in {}''' + days = ",".join(days) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testFormattingListComprehensions(self): + code = textwrap.dedent("""\ + def a(): + if True: + if True: + if True: + columns = [ + x for x, y in self._heap_this_is_very_long if x.route[0] == choice + ] + self._heap = [x for x in self._heap if x.route and x.route[0] == choice] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoSplittingWhenBinPacking(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, indent_width: 2, ' + 'continuation_indent_width: 4, indent_dictionary_value: True, ' + 'dedent_closing_brackets: True, ' + 'split_before_named_assigns: False}')) + code = textwrap.dedent("""\ + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4, + ) + + a_very_long_function_name( + long_argument_name_1=1, long_argument_name_2=2, long_argument_name_3=3, + long_argument_name_4=4 + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + + uwlines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testNotSplittingAfterSubscript(self): + unformatted_code = textwrap.dedent("""\ + if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b(c == d[ + 'eeeeee']).ffffff(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if not aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.b( + c == d['eeeeee']).ffffff(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingOneArgumentList(self): + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if True: + if True: + if True: + boxes[id_] = np.concatenate((points.min(axis=0), qoints.max(axis=0))) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + if True: + if True: + if True: + if True: + if True: + boxes[id_] = np.concatenate((points.min(axis=0), + qoints.max(axis=0))) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingBeforeFirstElementListArgument(self): + unformatted_code = textwrap.dedent("""\ + class _(): + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + if True: + if True: + if True: + return cls._create_investigation_result( + ( + clue for clue in combination if not clue == Verifier.UNMATCHED + ), constraints, InvestigationResult.OR + ) + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + if True: + if True: + if True: + return cls._create_investigation_result( + (clue for clue in combination if not clue == Verifier.UNMATCHED), + constraints, InvestigationResult.OR) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingArgumentsTerminatedByComma(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, ' + 'split_arguments_when_comma_terminated: True}')) + unformatted_code = textwrap.dedent("""\ + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3) + + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3,) + + a_very_long_function_name(long_argument_name_1=1, long_argument_name_2=2, long_argument_name_3=3, long_argument_name_4=4) + + a_very_long_function_name(long_argument_name_1, long_argument_name_2, long_argument_name_3, long_argument_name_4,) + + r =f0 (1, 2,3,) + """) + expected_formatted_code = textwrap.dedent("""\ + function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3) + + function_name( + argument_name_1=1, + argument_name_2=2, + argument_name_3=3, + ) + + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4) + + a_very_long_function_name( + long_argument_name_1, + long_argument_name_2, + long_argument_name_3, + long_argument_name_4, + ) + + r = f0( + 1, + 2, + 3, + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + + uwlines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(expected_formatted_code, reformatted_code) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testImportAsList(self): + code = textwrap.dedent("""\ + from toto import titi, tata, tutu # noqa + from toto import titi, tata, tutu + from toto import (titi, tata, tutu) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDictionaryValuesOnOwnLines(self): + unformatted_code = textwrap.dedent("""\ + a = { + 'aaaaaaaaaaaaaaaaaaaaaaaa': + Check('ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ', '=', True), + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb': + Check('YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', '=', True), + 'ccccccccccccccc': + Check('XXXXXXXXXXXXXXXXXXX', '!=', 'SUSPENDED'), + 'dddddddddddddddddddddddddddddd': + Check('WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW', '=', False), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeee': + Check('VVVVVVVVVVVVVVVVVVVVVVVVVVVVVV', '=', False), + 'ffffffffffffffffffffffffff': + Check('UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU', '=', True), + 'ggggggggggggggggg': + Check('TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT', '=', True), + 'hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh': + Check('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS', '=', True), + 'iiiiiiiiiiiiiiiiiiiiiiii': + Check('RRRRRRRRRRRRRRRRRRRRRRRRRRR', '=', True), + 'jjjjjjjjjjjjjjjjjjjjjjjjjj': + Check('QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ', '=', False), + } + """) + expected_formatted_code = textwrap.dedent("""\ + a = { + 'aaaaaaaaaaaaaaaaaaaaaaaa': + Check('ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ', '=', True), + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb': + Check('YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY', '=', True), + 'ccccccccccccccc': + Check('XXXXXXXXXXXXXXXXXXX', '!=', 'SUSPENDED'), + 'dddddddddddddddddddddddddddddd': + Check('WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW', '=', False), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeee': + Check('VVVVVVVVVVVVVVVVVVVVVVVVVVVVVV', '=', False), + 'ffffffffffffffffffffffffff': + Check('UUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU', '=', True), + 'ggggggggggggggggg': + Check('TTTTTTTTTTTTTTTTTTTTTTTTTTTTTT', '=', True), + 'hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh': + Check('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS', '=', True), + 'iiiiiiiiiiiiiiiiiiiiiiii': + Check('RRRRRRRRRRRRRRRRRRRRRRRRRRR', '=', True), + 'jjjjjjjjjjjjjjjjjjjjjjjjjj': + Check('QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ', '=', False), + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDictionaryOnOwnLine(self): + unformatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={ 'a': 'b' }, + branch_key=branch.key, + collection_key=collection.key) + """) + expected_formatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={'a': 'b'}, branch_key=branch.key, collection_key=collection.key) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={ 'a': 'b' }, + branch_key=branch.key, + collection_key=collection.key, + collection_key2=collection.key2) + """) + expected_formatted_code = textwrap.dedent("""\ + doc = test_utils.CreateTestDocumentViaController( + content={'a': 'b'}, + branch_key=branch.key, + collection_key=collection.key, + collection_key2=collection.key2) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNestedListsInDictionary(self): + unformatted_code = textwrap.dedent("""\ + _A = { + 'cccccccccc': ('^^1',), + 'rrrrrrrrrrrrrrrrrrrrrrrrr': ('^7913', # AAAAAAAAAAAAAA. + ), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee': ('^6242', # BBBBBBBBBBBBBBB. + ), + 'vvvvvvvvvvvvvvvvvvv': ('^27959', # CCCCCCCCCCCCCCCCCC. + '^19746', # DDDDDDDDDDDDDDDDDDDDDDD. + '^22907', # EEEEEEEEEEEEEEEEEEEEEEEE. + '^21098', # FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF. + '^22826', # GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG. + '^22769', # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. + '^22935', # IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII. + '^3982', # JJJJJJJJJJJJJ. + ), + 'uuuuuuuuuuuu': ('^19745', # LLLLLLLLLLLLLLLLLLLLLLLLLL. + '^21324', # MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM. + '^22831', # NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN. + '^17081', # OOOOOOOOOOOOOOOOOOOOO. + ), + 'eeeeeeeeeeeeee': ( + '^9416', # Reporter email. Not necessarily the reporter. + '^^3', # This appears to be the raw email field. + ), + 'cccccccccc': ('^21109', # PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP. + ), + } + """) + expected_formatted_code = textwrap.dedent("""\ + _A = { + 'cccccccccc': ('^^1',), + 'rrrrrrrrrrrrrrrrrrrrrrrrr': ( + '^7913', # AAAAAAAAAAAAAA. + ), + 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee': ( + '^6242', # BBBBBBBBBBBBBBB. + ), + 'vvvvvvvvvvvvvvvvvvv': ( + '^27959', # CCCCCCCCCCCCCCCCCC. + '^19746', # DDDDDDDDDDDDDDDDDDDDDDD. + '^22907', # EEEEEEEEEEEEEEEEEEEEEEEE. + '^21098', # FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF. + '^22826', # GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG. + '^22769', # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. + '^22935', # IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII. + '^3982', # JJJJJJJJJJJJJ. + ), + 'uuuuuuuuuuuu': ( + '^19745', # LLLLLLLLLLLLLLLLLLLLLLLLLL. + '^21324', # MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM. + '^22831', # NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN. + '^17081', # OOOOOOOOOOOOOOOOOOOOO. + ), + 'eeeeeeeeeeeeee': ( + '^9416', # Reporter email. Not necessarily the reporter. + '^^3', # This appears to be the raw email field. + ), + 'cccccccccc': ( + '^21109', # PPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP. + ), + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNestedDictionary(self): + unformatted_code = textwrap.dedent("""\ + class _(): + def _(): + breadcrumbs = [{'name': 'Admin', + 'url': url_for(".home")}, + {'title': title},] + breadcrumbs = [{'name': 'Admin', + 'url': url_for(".home")}, + {'title': title}] + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + def _(): + breadcrumbs = [ + { + 'name': 'Admin', + 'url': url_for(".home") + }, + { + 'title': title + }, + ] + breadcrumbs = [{'name': 'Admin', 'url': url_for(".home")}, {'title': title}] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDictionaryElementsOnOneLine(self): + code = textwrap.dedent("""\ + class _(): + + @mock.patch.dict( + os.environ, + {'HTTP_' + xsrf._XSRF_TOKEN_HEADER.replace('-', '_'): 'atoken'}) + def _(): + pass + + + AAAAAAAAAAAAAAAAAAAAAAAA = { + Environment.XXXXXXXXXX: 'some text more text even more tex', + Environment.YYYYYYY: 'some text more text even more text yet ag', + Environment.ZZZZZZZZZZZ: 'some text more text even mor etext yet again tex', + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNotInParams(self): + unformatted_code = textwrap.dedent("""\ + list("a long line to break the line. a long line to break the brk a long lin", not True) + """) + expected_code = textwrap.dedent("""\ + list("a long line to break the line. a long line to break the brk a long lin", + not True) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testNamedAssignNotAtEndOfLine(self): + unformatted_code = textwrap.dedent("""\ + def _(): + if True: + with py3compat.open_with_encoding(filename, mode='w', + encoding=encoding) as fd: + pass + """) + expected_code = textwrap.dedent("""\ + def _(): + if True: + with py3compat.open_with_encoding( + filename, mode='w', encoding=encoding) as fd: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testBlankLineBeforeClassDocstring(self): + unformatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + expected_code = textwrap.dedent('''\ + class A: + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, ' + 'blank_line_before_class_docstring: True}')) + unformatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + class A: + + """Does something. + + Also, here are some details. + """ + + def __init__(self): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testBlankLineBeforeModuleDocstring(self): + unformatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + + """Some module docstring.""" + + + def foobar(): + pass + ''') + expected_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + """Some module docstring.""" + + + def foobar(): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'blank_line_before_module_docstring: True}')) + unformatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + """Some module docstring.""" + + + def foobar(): + pass + ''') + expected_formatted_code = textwrap.dedent('''\ + #!/usr/bin/env python + # -*- coding: utf-8 name> -*- + + """Some module docstring.""" + + + def foobar(): + pass + ''') + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testTupleCohesion(self): + unformatted_code = textwrap.dedent("""\ + def f(): + this_is_a_very_long_function_name(an_extremely_long_variable_name, ( + 'a string that may be too long %s' % 'M15')) + """) + expected_code = textwrap.dedent("""\ + def f(): + this_is_a_very_long_function_name( + an_extremely_long_variable_name, + ('a string that may be too long %s' % 'M15')) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testSubscriptExpression(self): + code = textwrap.dedent("""\ + foo = d[not a] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testListWithFunctionCalls(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + return [ + Bar( + xxx='some string', + yyy='another long string', + zzz='a third long string'), Bar( + xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + expected_code = textwrap.dedent("""\ + def foo(): + return [ + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string'), + Bar(xxx='some string', + yyy='another long string', + zzz='a third long string') + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testEllipses(self): + unformatted_code = textwrap.dedent("""\ + X=... + Y = X if ... else X + """) + expected_code = textwrap.dedent("""\ + X = ... + Y = X if ... else X + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertEqual(expected_code, reformatter.Reformat(uwlines)) + + def testSplittingBeforeFirstArgumentOnFunctionCall(self): + """Tests split_before_first_argument on a function call.""" + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, split_before_first_argument: True}')) + unformatted_code = textwrap.dedent("""\ + a_very_long_function_name("long string with formatting {0:s}".format( + "mystring")) + """) + expected_formatted_code = textwrap.dedent("""\ + a_very_long_function_name( + "long string with formatting {0:s}".format("mystring")) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testSplittingBeforeFirstArgumentOnFunctionDefinition(self): + """Tests split_before_first_argument on a function definition.""" + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, split_before_first_argument: True}')) + unformatted_code = textwrap.dedent("""\ + def _GetNumberOfSecondsFromElements(year, month, day, hours, + minutes, seconds, microseconds): + return + """) + expected_formatted_code = textwrap.dedent("""\ + def _GetNumberOfSecondsFromElements( + year, month, day, hours, minutes, seconds, microseconds): + return + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testSplittingBeforeFirstArgumentOnCompoundStatement(self): + """Tests split_before_first_argument on a compound statement.""" + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, split_before_first_argument: True}')) + unformatted_code = textwrap.dedent("""\ + if (long_argument_name_1 == 1 or + long_argument_name_2 == 2 or + long_argument_name_3 == 3 or + long_argument_name_4 == 4): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if (long_argument_name_1 == 1 or long_argument_name_2 == 2 or + long_argument_name_3 == 3 or long_argument_name_4 == 4): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testCoalesceBracketsOnDict(self): + """Tests coalesce_brackets on a dictionary.""" + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, coalesce_brackets: True}')) + unformatted_code = textwrap.dedent("""\ + date_time_values = ( + { + u'year': year, + u'month': month, + u'day_of_month': day_of_month, + u'hours': hours, + u'minutes': minutes, + u'seconds': seconds + } + ) + """) + expected_formatted_code = textwrap.dedent("""\ + date_time_values = ({ + u'year': year, + u'month': month, + u'day_of_month': day_of_month, + u'hours': hours, + u'minutes': minutes, + u'seconds': seconds + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testSplitAfterComment(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: chromium, coalesce_brackets: True, ' + 'dedent_closing_brackets: true}')) + code = textwrap.dedent("""\ + if __name__ == "__main__": + with another_resource: + account = { + "validUntil": + int(time() + (6 * 7 * 24 * 60 * 60)) # in 6 weeks time + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testAsyncAsNonKeyword(self): + try: + style.SetGlobalStyle(style.CreatePEP8Style()) + + # In Python 2, async may be used as a non-keyword identifier. + code = textwrap.dedent("""\ + from util import async + + + class A(object): + def foo(self): + async.run() + + def bar(self): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines, verify=False)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testDisableEndingCommaHeuristic(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: chromium,' + ' disable_ending_comma_heuristic: True}')) + + code = """\ +x = [1, 2, 3, 4, 5, 6, 7,] +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_buganizer_test.py b/yapftests/reformatter_buganizer_test.py new file mode 100644 index 0000000..6a1c781 --- /dev/null +++ b/yapftests/reformatter_buganizer_test.py @@ -0,0 +1,1959 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Buganizer tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class BuganizerFixes(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testB77923341(self): + code = """\ +def f(): + if (aaaaaaaaaaaaaa.bbbbbbbbbbbb.ccccc <= 0 and # pytype: disable=attribute-error + ddddddddddd.eeeeeeeee == constants.FFFFFFFFFFFFFF): + raise "yo" +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB77329955(self): + code = """\ +class _(): + + @parameterized.named_parameters( + ('ReadyExpiredSuccess', True, True, True, None, None), + ('SpannerUpdateFails', True, False, True, None, None), + ('ReadyNotExpired', False, True, True, True, None), + # ('ReadyNotExpiredNotHealthy', False, True, True, False, True), + # ('ReadyNotExpiredNotHealthyErrorFails', False, True, True, False, False + # ('ReadyNotExpiredNotHealthyUpdateFails', False, False, True, False, True + ) + def _(): + pass +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB65197969(self): + unformatted_code = """\ +class _(): + + def _(): + return timedelta(seconds=max(float(time_scale), small_interval) * + 1.41 ** min(num_attempts, 9)) +""" + expected_formatted_code = """\ +class _(): + + def _(): + return timedelta( + seconds=max(float(time_scale), small_interval) * + 1.41**min(num_attempts, 9)) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB65546221(self): + unformatted_code = """\ +SUPPORTED_PLATFORMS = ( + "centos-6", + "centos-7", + "ubuntu-1204-precise", + "ubuntu-1404-trusty", + "ubuntu-1604-xenial", + "debian-7-wheezy", + "debian-8-jessie", + "debian-9-stretch",) +""" + expected_formatted_code = """\ +SUPPORTED_PLATFORMS = ( + "centos-6", + "centos-7", + "ubuntu-1204-precise", + "ubuntu-1404-trusty", + "ubuntu-1604-xenial", + "debian-7-wheezy", + "debian-8-jessie", + "debian-9-stretch", +) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30500455(self): + unformatted_code = """\ +INITIAL_SYMTAB = dict([(name, 'exception#' + name) for name in INITIAL_EXCEPTIONS +] * [(name, 'type#' + name) for name in INITIAL_TYPES] + [ + (name, 'function#' + name) for name in INITIAL_FUNCTIONS +] + [(name, 'const#' + name) for name in INITIAL_CONSTS]) +""" + expected_formatted_code = """\ +INITIAL_SYMTAB = dict( + [(name, 'exception#' + name) for name in INITIAL_EXCEPTIONS] * + [(name, 'type#' + name) for name in INITIAL_TYPES] + + [(name, 'function#' + name) for name in INITIAL_FUNCTIONS] + + [(name, 'const#' + name) for name in INITIAL_CONSTS]) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB38343525(self): + code = """\ +# This does foo. +@arg.String('some_path_to_a_file', required=True) +# This does bar. +@arg.String('some_path_to_a_file', required=True) +def f(): + print 1 +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB37099651(self): + unformatted_code = """\ +_MEMCACHE = lazy.MakeLazy( + # pylint: disable=g-long-lambda + lambda: function.call.mem.clients(FLAGS.some_flag_thingy, default_namespace=_LAZY_MEM_NAMESPACE, allow_pickle=True) + # pylint: enable=g-long-lambda +) +""" + expected_formatted_code = """\ +_MEMCACHE = lazy.MakeLazy( + # pylint: disable=g-long-lambda + lambda: function.call.mem.clients( + FLAGS.some_flag_thingy, + default_namespace=_LAZY_MEM_NAMESPACE, + allow_pickle=True) + # pylint: enable=g-long-lambda +) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB33228502(self): + unformatted_code = """\ +def _(): + success_rate_stream_table = module.Precompute( + query_function=module.DefineQueryFunction( + name='Response error ratio', + expression=((m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), + {'borg_job': module_config.job, 'metric:response_type': 'SUCCESS'}), + m.Fetch(m.Raw('monarch.BorgTask', '/corp/travel/trips2/dispatcher/email/response'), {'borg_job': module_config.job})) + | m.Window(m.Delta('1h')) + | m.Join('successes', 'total') + | m.Point(m.VAL['successes'] / m.VAL['total'])))) +""" + expected_formatted_code = """\ +def _(): + success_rate_stream_table = module.Precompute( + query_function=module.DefineQueryFunction( + name='Response error ratio', + expression=( + (m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), { + 'borg_job': module_config.job, + 'metric:response_type': 'SUCCESS' + }), + m.Fetch( + m.Raw('monarch.BorgTask', + '/corp/travel/trips2/dispatcher/email/response'), + {'borg_job': module_config.job})) + | m.Window(m.Delta('1h')) + | m.Join('successes', 'total') + | m.Point(m.VAL['successes'] / m.VAL['total'])))) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30394228(self): + code = """\ +class _(): + + def _(self): + return some.randome.function.calling( + wf, None, alert.Format(alert.subject, alert=alert, threshold=threshold), + alert.Format(alert.body, alert=alert, threshold=threshold), + alert.html_formatting) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB65246454(self): + unformatted_code = """\ +class _(): + + def _(self): + self.assertEqual({i.id + for i in successful_instances}, + {i.id + for i in self._statuses.successful_instances}) +""" + expected_formatted_code = """\ +class _(): + + def _(self): + self.assertEqual({i.id for i in successful_instances}, + {i.id for i in self._statuses.successful_instances}) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB67935450(self): + unformatted_code = """\ +def _(): + return ( + (Gauge( + metric='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re}), + Gauge( + metric='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re})) + | expr.Join( + left_name='start', left_default=0, right_name='end', right_default=0) + | m.Point( + m.Cond(m.VAL['end'] != 0, m.VAL['end'], k.TimestampMicros() / + 1000000L) - m.Cond(m.VAL['start'] != 0, m.VAL['start'], + m.TimestampMicros() / 1000000L))) +""" + expected_formatted_code = """\ +def _(): + return ( + (Gauge( + metric='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re}), + Gauge( + metric='bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + group_by=group_by + ['metric:process_name'], + metric_filter={'metric:process_name': process_name_re})) + | expr.Join( + left_name='start', left_default=0, right_name='end', right_default=0) + | m.Point( + m.Cond(m.VAL['end'] != 0, m.VAL['end'], + k.TimestampMicros() / 1000000L) - + m.Cond(m.VAL['start'] != 0, m.VAL['start'], + m.TimestampMicros() / 1000000L))) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB66011084(self): + unformatted_code = """\ +X = { +"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": # Comment 1. +([] if True else [ # Comment 2. + "bbbbbbbbbbbbbbbbbbb", # Comment 3. + "cccccccccccccccccccccccc", # Comment 4. + "ddddddddddddddddddddddddd", # Comment 5. + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", # Comment 6. + "fffffffffffffffffffffffffffffff", # Comment 7. + "ggggggggggggggggggggggggggg", # Comment 8. + "hhhhhhhhhhhhhhhhhh", # Comment 9. +]), +} +""" + expected_formatted_code = """\ +X = { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": # Comment 1. + ([] if True else [ # Comment 2. + "bbbbbbbbbbbbbbbbbbb", # Comment 3. + "cccccccccccccccccccccccc", # Comment 4. + "ddddddddddddddddddddddddd", # Comment 5. + "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", # Comment 6. + "fffffffffffffffffffffffffffffff", # Comment 7. + "ggggggggggggggggggggggggggg", # Comment 8. + "hhhhhhhhhhhhhhhhhh", # Comment 9. + ]), +} +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB67455376(self): + unformatted_code = """\ +sponge_ids.extend(invocation.id() for invocation in self._client.GetInvocationsByLabels(labels)) +""" + expected_formatted_code = """\ +sponge_ids.extend(invocation.id() + for invocation in self._client.GetInvocationsByLabels(labels)) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB35210351(self): + unformatted_code = """\ +def _(): + config.AnotherRuleThing( + 'the_title_to_the_thing_here', + {'monitorname': 'firefly', + 'service': ACCOUNTING_THING, + 'severity': 'the_bug', + 'monarch_module_name': alerts.TheLabel(qa_module_regexp, invert=True)}, + fanout, + alerts.AlertUsToSomething( + GetTheAlertToIt('the_title_to_the_thing_here'), + GetNotificationTemplate('your_email_here'))) +""" + expected_formatted_code = """\ +def _(): + config.AnotherRuleThing( + 'the_title_to_the_thing_here', { + 'monitorname': 'firefly', + 'service': ACCOUNTING_THING, + 'severity': 'the_bug', + 'monarch_module_name': alerts.TheLabel(qa_module_regexp, invert=True) + }, fanout, + alerts.AlertUsToSomething( + GetTheAlertToIt('the_title_to_the_thing_here'), + GetNotificationTemplate('your_email_here'))) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB34774905(self): + unformatted_code = """\ +x=[VarExprType(ir_name=IrName( value='x', +expr_type=UnresolvedAttrExprType( atom=UnknownExprType(), attr_name=IrName( + value='x', expr_type=UnknownExprType(), usage='UNKNOWN', fqn=None, + astn=None), usage='REF'), usage='ATTR', fqn='<attr>.x', astn=None))] +""" + expected_formatted_code = """\ +x = [ + VarExprType( + ir_name=IrName( + value='x', + expr_type=UnresolvedAttrExprType( + atom=UnknownExprType(), + attr_name=IrName( + value='x', + expr_type=UnknownExprType(), + usage='UNKNOWN', + fqn=None, + astn=None), + usage='REF'), + usage='ATTR', + fqn='<attr>.x', + astn=None)) +] +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB65176185(self): + code = """\ +xx = zip(*[(a, b) for (a, b, c) in yy]) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB35210166(self): + unformatted_code = """\ +def _(): + query = ( + m.Fetch(n.Raw('monarch.BorgTask', '/proc/container/memory/usage'), { 'borg_user': borguser, 'borg_job': jobname }) + | o.Window(m.Align('5m')) | p.GroupBy(['borg_user', 'borg_job', 'borg_cell'], q.Mean())) +""" + expected_formatted_code = """\ +def _(): + query = ( + m.Fetch( + n.Raw('monarch.BorgTask', '/proc/container/memory/usage'), { + 'borg_user': borguser, + 'borg_job': jobname + }) + | o.Window(m.Align('5m')) + | p.GroupBy(['borg_user', 'borg_job', 'borg_cell'], q.Mean())) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB32167774(self): + unformatted_code = """\ +X = ( + 'is_official', + 'is_cover', + 'is_remix', + 'is_instrumental', + 'is_live', + 'has_lyrics', + 'is_album', + 'is_compilation',) +""" + expected_formatted_code = """\ +X = ( + 'is_official', + 'is_cover', + 'is_remix', + 'is_instrumental', + 'is_live', + 'has_lyrics', + 'is_album', + 'is_compilation', +) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB66912275(self): + unformatted_code = """\ +def _(): + with self.assertRaisesRegexp(errors.HttpError, 'Invalid'): + patch_op = api_client.forwardingRules().patch( + project=project_id, + region=region, + forwardingRule=rule_name, + body={'fingerprint': base64.urlsafe_b64encode('invalid_fingerprint')}).execute() +""" + expected_formatted_code = """\ +def _(): + with self.assertRaisesRegexp(errors.HttpError, 'Invalid'): + patch_op = api_client.forwardingRules().patch( + project=project_id, + region=region, + forwardingRule=rule_name, + body={ + 'fingerprint': base64.urlsafe_b64encode('invalid_fingerprint') + }).execute() +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB67312284(self): + code = """\ +def _(): + self.assertEqual( + [u'to be published 2', u'to be published 1', u'to be published 0'], + [el.text for el in page.first_column_tds]) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB65241516(self): + unformatted_code = """\ +checkpoint_files = gfile.Glob(os.path.join(TrainTraceDir(unit_key, "*", "*"), embedding_model.CHECKPOINT_FILENAME + "-*")) +""" + expected_formatted_code = """\ +checkpoint_files = gfile.Glob( + os.path.join( + TrainTraceDir(unit_key, "*", "*"), + embedding_model.CHECKPOINT_FILENAME + "-*")) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB37460004(self): + code = textwrap.dedent("""\ + assert all(s not in (_SENTINEL, None) for s in nested_schemas + ), 'Nested schemas should never contain None/_SENTINEL' + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB36806207(self): + code = """\ +def _(): + linearity_data = [[row] for row in [ + "%.1f mm" % (np.mean(linearity_values["pos_error"]) * 1000.0), + "%.1f mm" % (np.max(linearity_values["pos_error"]) * 1000.0), + "%.1f mm" % (np.mean(linearity_values["pos_error_chunk_mean"]) * 1000.0), + "%.1f mm" % (np.max(linearity_values["pos_error_chunk_max"]) * 1000.0), + "%.1f deg" % math.degrees(np.mean(linearity_values["rot_noise"])), + "%.1f deg" % math.degrees(np.max(linearity_values["rot_noise"])), + "%.1f deg" % math.degrees(np.mean(linearity_values["rot_drift"])), + "%.1f deg" % math.degrees(np.max(linearity_values["rot_drift"])), + "%.1f%%" % (np.max(linearity_values["pos_discontinuity"]) * 100.0), + "%.1f%%" % (np.max(linearity_values["rot_discontinuity"]) * 100.0) + ]] +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB36215507(self): + code = textwrap.dedent("""\ + class X(): + + def _(): + aaaaaaaaaaaaa._bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb( + mmmmmmmmmmmmm, nnnnn, ooooooooo, + _(ppppppppppppppppppppppppppppppppppppp), + *(qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq), + **(qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB35212469(self): + unformatted_code = textwrap.dedent("""\ + def _(): + X = { + 'retain': { + 'loadtest': # This is a comment in the middle of a dictionary entry + ('/some/path/to/a/file/that/is/needed/by/this/process') + } + } + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + X = { + 'retain': { + 'loadtest': # This is a comment in the middle of a dictionary entry + ('/some/path/to/a/file/that/is/needed/by/this/process') + } + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB31063453(self): + unformatted_code = textwrap.dedent("""\ + def _(): + while ((not mpede_proc) or ((time_time() - last_modified) < FLAGS_boot_idle_timeout)): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + while ((not mpede_proc) or + ((time_time() - last_modified) < FLAGS_boot_idle_timeout)): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB35021894(self): + unformatted_code = textwrap.dedent("""\ + def _(): + labelacl = Env(qa={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }, + prod={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + labelacl = Env( + qa={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }, + prod={ + 'read': 'name/some-type-of-very-long-name-for-reading-perms', + 'modify': 'name/some-other-type-of-very-long-name-for-modifying' + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB34682902(self): + unformatted_code = textwrap.dedent("""\ + logging.info("Mean angular velocity norm: %.3f", np.linalg.norm(np.mean(ang_vel_arr, axis=0))) + """) + expected_formatted_code = textwrap.dedent("""\ + logging.info("Mean angular velocity norm: %.3f", + np.linalg.norm(np.mean(ang_vel_arr, axis=0))) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB33842726(self): + unformatted_code = textwrap.dedent("""\ + class _(): + def _(): + hints.append(('hg tag -f -l -r %s %s # %s' % (short(ctx.node( + )), candidatetag, firstline))[:78]) + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + def _(): + hints.append(('hg tag -f -l -r %s %s # %s' % (short( + ctx.node()), candidatetag, firstline))[:78]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB32931780(self): + unformatted_code = textwrap.dedent("""\ + environments = { + 'prod': { + # this is a comment before the first entry. + 'entry one': + 'an entry.', + # this is the comment before the second entry. + 'entry number 2.': + 'something', + # this is the comment before the third entry and it's a doozy. So big! + 'who': + 'allin', + # This is an entry that has a dictionary in it. It's ugly + 'something': { + 'page': ['this-is-a-page@xxxxxxxx.com', 'something-for-eml@xxxxxx.com'], + 'bug': ['bugs-go-here5300@xxxxxx.com'], + 'email': ['sometypeof-email@xxxxxx.com'], + }, + # a short comment + 'yolo!!!!!': + 'another-email-address@xxxxxx.com', + # this entry has an implicit string concatenation + 'implicit': + 'https://this-is-very-long.url-addr.com/' + '?something=something%20some%20more%20stuff..', + # A more normal entry. + '.....': + 'this is an entry', + } + } + """) + expected_formatted_code = textwrap.dedent("""\ + environments = { + 'prod': { + # this is a comment before the first entry. + 'entry one': 'an entry.', + # this is the comment before the second entry. + 'entry number 2.': 'something', + # this is the comment before the third entry and it's a doozy. So big! + 'who': 'allin', + # This is an entry that has a dictionary in it. It's ugly + 'something': { + 'page': [ + 'this-is-a-page@xxxxxxxx.com', 'something-for-eml@xxxxxx.com' + ], + 'bug': ['bugs-go-here5300@xxxxxx.com'], + 'email': ['sometypeof-email@xxxxxx.com'], + }, + # a short comment + 'yolo!!!!!': 'another-email-address@xxxxxx.com', + # this entry has an implicit string concatenation + 'implicit': 'https://this-is-very-long.url-addr.com/' + '?something=something%20some%20more%20stuff..', + # A more normal entry. + '.....': 'this is an entry', + } + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB33047408(self): + code = textwrap.dedent("""\ + def _(): + for sort in (sorts or []): + request['sorts'].append({ + 'field': { + 'user_field': sort + }, + 'order': 'ASCENDING' + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB32714745(self): + code = textwrap.dedent("""\ + class _(): + + def _BlankDefinition(): + '''Return a generic blank dictionary for a new field.''' + return { + 'type': '', + 'validation': '', + 'name': 'fieldname', + 'label': 'Field Label', + 'help': '', + 'initial': '', + 'required': False, + 'required_msg': 'Required', + 'invalid_msg': 'Please enter a valid value', + 'options': { + 'regex': '', + 'widget_attr': '', + 'choices_checked': '', + 'choices_count': '', + 'choices': {} + }, + 'isnew': True, + 'dirty': False, + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB32737279(self): + unformatted_code = textwrap.dedent("""\ + here_is_a_dict = { + 'key': + # Comment. + 'value' + } + """) + expected_formatted_code = textwrap.dedent("""\ + here_is_a_dict = { + 'key': # Comment. + 'value' + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB32570937(self): + code = textwrap.dedent("""\ + def _(): + if (job_message.ball not in ('*', ball) or + job_message.call not in ('*', call) or + job_message.mall not in ('*', job_name)): + return False + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB31937033(self): + code = textwrap.dedent("""\ + class _(): + + def __init__(self, metric, fields_cb=None): + self._fields_cb = fields_cb or (lambda *unused_args, **unused_kwargs: {}) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB31911533(self): + code = """\ +class _(): + + @parameterized.NamedParameters( + ('IncludingModInfoWithHeaderList', AAAA, aaaa), + ('IncludingModInfoWithoutHeaderList', BBBB, bbbbb), + ('ExcludingModInfoWithHeaderList', CCCCC, cccc), + ('ExcludingModInfoWithoutHeaderList', DDDDD, ddddd), + ) + def _(): + pass +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB31847238(self): + unformatted_code = textwrap.dedent("""\ + class _(): + + def aaaaa(self, bbbbb, cccccccccccccc=None): # TODO(who): pylint: disable=unused-argument + return 1 + + def xxxxx(self, yyyyy, zzzzzzzzzzzzzz=None): # A normal comment that runs over the column limit. + return 1 + """) + expected_formatted_code = textwrap.dedent("""\ + class _(): + + def aaaaa(self, bbbbb, cccccccccccccc=None): # TODO(who): pylint: disable=unused-argument + return 1 + + def xxxxx( + self, yyyyy, + zzzzzzzzzzzzzz=None): # A normal comment that runs over the column limit. + return 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30760569(self): + unformatted_code = textwrap.dedent("""\ + {'1234567890123456789012345678901234567890123456789012345678901234567890': + '1234567890123456789012345678901234567890'} + """) + expected_formatted_code = textwrap.dedent("""\ + { + '1234567890123456789012345678901234567890123456789012345678901234567890': + '1234567890123456789012345678901234567890' + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB26034238(self): + unformatted_code = textwrap.dedent("""\ + class Thing: + + def Function(self): + thing.Scrape('/aaaaaaaaa/bbbbbbbbbb/ccccc/dddd/eeeeeeeeeeeeee/ffffffffffffff').AndReturn(42) + """) + expected_formatted_code = textwrap.dedent("""\ + class Thing: + + def Function(self): + thing.Scrape( + '/aaaaaaaaa/bbbbbbbbbb/ccccc/dddd/eeeeeeeeeeeeee/ffffffffffffff' + ).AndReturn(42) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30536435(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + if True: + aaaaaaaaaaa.comment('import-from[{}] {} {}'.format( + bbbbbbbbb.usage, + ccccccccc.within, + imports.ddddddddddddddddddd(name_item.ffffffffffffffff))) + """) + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + if True: + aaaaaaaaaaa.comment('import-from[{}] {} {}'.format( + bbbbbbbbb.usage, ccccccccc.within, + imports.ddddddddddddddddddd(name_item.ffffffffffffffff))) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30442148(self): + unformatted_code = textwrap.dedent("""\ + def lulz(): + return (some_long_module_name.SomeLongClassName. + some_long_attribute_name.some_long_method_name()) + """) + expected_formatted_code = textwrap.dedent("""\ + def lulz(): + return (some_long_module_name.SomeLongClassName.some_long_attribute_name. + some_long_method_name()) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB26868213(self): + unformatted_code = textwrap.dedent("""\ + def _(): + xxxxxxxxxxxxxxxxxxx = { + 'ssssss': {'ddddd': 'qqqqq', + 'p90': aaaaaaaaaaaaaaaaa, + 'p99': bbbbbbbbbbbbbbbbb, + 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(),}, + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { + 'ddddd': 'bork bork bork bo', + 'p90': wwwwwwwwwwwwwwwww, + 'p99': wwwwwwwwwwwwwwwww, + 'lllllllllllll': None, # use the default + } + } + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + xxxxxxxxxxxxxxxxxxx = { + 'ssssss': { + 'ddddd': 'qqqqq', + 'p90': aaaaaaaaaaaaaaaaa, + 'p99': bbbbbbbbbbbbbbbbb, + 'lllllllllllll': yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy(), + }, + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbb': { + 'ddddd': 'bork bork bork bo', + 'p90': wwwwwwwwwwwwwwwww, + 'p99': wwwwwwwwwwwwwwwww, + 'lllllllllllll': None, # use the default + } + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB30173198(self): + code = textwrap.dedent("""\ + class _(): + + def _(): + self.assertFalse( + evaluation_runner.get_larps_in_eval_set('these_arent_the_larps')) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB29908765(self): + code = textwrap.dedent("""\ + class _(): + + def __repr__(self): + return '<session %s on %s>' % (self._id, + self._stub._stub.rpc_channel().target()) # pylint:disable=protected-access + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB30087362(self): + code = textwrap.dedent("""\ + def _(): + for s in sorted(env['foo']): + bar() + # This is a comment + + # This is another comment + foo() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB30087363(self): + code = textwrap.dedent("""\ + if False: + bar() + # This is a comment + # This is another comment + elif True: + foo() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB29093579(self): + unformatted_code = textwrap.dedent("""\ + def _(): + _xxxxxxxxxxxxxxx(aaaaaaaa, bbbbbbbbbbbbbb.cccccccccc[ + dddddddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + _xxxxxxxxxxxxxxx( + aaaaaaaa, + bbbbbbbbbbbbbb.cccccccccc[dddddddddddddddddddddddddddd. + eeeeeeeeeeeeeeeeeeeeee.fffffffffffffffffffff]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB26382315(self): + code = textwrap.dedent("""\ + @hello_world + # This is a first comment + + # Comment + def foo(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB27616132(self): + unformatted_code = textwrap.dedent("""\ + if True: + query.fetch_page.assert_has_calls([ + mock.call(100, + start_cursor=None), + mock.call(100, + start_cursor=cursor_1), + mock.call(100, + start_cursor=cursor_2), + ]) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + query.fetch_page.assert_has_calls([ + mock.call(100, start_cursor=None), + mock.call(100, start_cursor=cursor_1), + mock.call(100, start_cursor=cursor_2), + ]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB27590179(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( + { True: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), + False: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) + }) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + self.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ({ + True: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee), + False: + self.bbb.cccccccccc(ddddddddddddddddddddddd.eeeeeeeeeeeeeeeeeeeeee) + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB27266946(self): + unformatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = (self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccccccccc) + """) + expected_formatted_code = textwrap.dedent("""\ + def _(): + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa = ( + self.bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. + cccccccccccccccccccccccccccccccccccc) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB25505359(self): + code = textwrap.dedent("""\ + _EXAMPLE = { + 'aaaaaaaaaaaaaa': [{ + 'bbbb': 'cccccccccccccccccccccc', + 'dddddddddddd': [] + }, { + 'bbbb': 'ccccccccccccccccccc', + 'dddddddddddd': [] + }] + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB25324261(self): + code = textwrap.dedent("""\ + aaaaaaaaa = set(bbbb.cccc + for ddd in eeeeee.fffffffffff.gggggggggggggggg + for cccc in ddd.specification) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB25136704(self): + code = textwrap.dedent("""\ + class f: + + def test(self): + self.bbbbbbb[0]['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', { + 'xxxxxx': 'yyyyyy' + }] = cccccc.ddd('1m', '10x1+1') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB25165602(self): + code = textwrap.dedent("""\ + def f(): + ids = {u: i for u, i in zip(self.aaaaa, xrange(42, 42 + len(self.aaaaaa)))} + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB25157123(self): + code = textwrap.dedent("""\ + def ListArgs(): + FairlyLongMethodName([relatively_long_identifier_for_a_list], + another_argument_with_a_long_identifier) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB25136820(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + return collections.OrderedDict({ + # Preceding comment. + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': + '$bbbbbbbbbbbbbbbbbbbbbbbb', + }) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return collections.OrderedDict({ + # Preceding comment. + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa': + '$bbbbbbbbbbbbbbbbbbbbbbbb', + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB25131481(self): + unformatted_code = textwrap.dedent("""\ + APPARENT_ACTIONS = ('command_type', { + 'materialize': lambda x: some_type_of_function('materialize ' + x.command_def), + '#': lambda x: x # do nothing + }) + """) + expected_formatted_code = textwrap.dedent("""\ + APPARENT_ACTIONS = ( + 'command_type', + { + 'materialize': + lambda x: some_type_of_function('materialize ' + x.command_def), + '#': + lambda x: x # do nothing + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB23445244(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if True: + return xxxxxxxxxxxxxxxx( + command, + extra_env={ + "OOOOOOOOOOOOOOOOOOOOO": FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, + "PPPPPPPPPPPPPPPPPPPPP": + FLAGS.aaaaaaaaaaaaaa + FLAGS.bbbbbbbbbbbbbbbbbbb, + }) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if True: + return xxxxxxxxxxxxxxxx( + command, + extra_env={ + "OOOOOOOOOOOOOOOOOOOOO": + FLAGS.zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz, + "PPPPPPPPPPPPPPPPPPPPP": + FLAGS.aaaaaaaaaaaaaa + FLAGS.bbbbbbbbbbbbbbbbbbb, + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB20559654(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + + def foo(self): + unused_error, result = server.Query( + ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], + aaaaaaaaaaa=True, bbbbbbbb=None) + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + + def foo(self): + unused_error, result = server.Query( + ['AA BBBB CCC DDD EEEEEEEE X YY ZZZZ FFF EEE AAAAAAAA'], + aaaaaaaaaaa=True, + bbbbbbbb=None) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB23943842(self): + unformatted_code = textwrap.dedent("""\ + class F(): + def f(): + self.assertDictEqual( + accounts, { + 'foo': + {'account': 'foo', + 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.'}, + 'bar': {'account': 'bar', + 'lines': 'l5\\nl6\\nl7'}, + 'wiz': {'account': 'wiz', + 'lines': 'l8'} + }) + """) + expected_formatted_code = textwrap.dedent("""\ + class F(): + + def f(): + self.assertDictEqual( + accounts, { + 'foo': { + 'account': 'foo', + 'lines': 'l1\\nl2\\nl3\\n1 line(s) were elided.' + }, + 'bar': { + 'account': 'bar', + 'lines': 'l5\\nl6\\nl7' + }, + 'wiz': { + 'account': 'wiz', + 'lines': 'l8' + } + }) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB20551180(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if True: + return (struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + eeeeeee) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if True: + return ( + struct.pack('aaaa', bbbbbbbbbb, ccccccccccccccc, dddddddd) + eeeeeee) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB23944849(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + def xxxxxxxxx(self, aaaaaaa, bbbbbbb=ccccccccccc, dddddd=300, eeeeeeeeeeeeee=None, fffffffffffffff=0): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + + def xxxxxxxxx(self, + aaaaaaa, + bbbbbbb=ccccccccccc, + dddddd=300, + eeeeeeeeeeeeee=None, + fffffffffffffff=0): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB23935890(self): + unformatted_code = textwrap.dedent("""\ + class F(): + def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, eeeeeeeeeeeeeee): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class F(): + + def functioni(self, aaaaaaa, bbbbbbb, cccccc, dddddddddddddd, + eeeeeeeeeeeeeee): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB28414371(self): + code = textwrap.dedent("""\ + def _(): + return ((m.fffff( + m.rrr('mmmmmmmmmmmmmmmm', 'ssssssssssssssssssssssssss'), ffffffffffffffff) + | m.wwwwww(m.ddddd('1h')) + | m.ggggggg(bbbbbbbbbbbbbbb) + | m.ppppp( + (1 - m.ffffffffffffffff(llllllllllllllllllllll * 1000000, m.vvv)) + * m.ddddddddddddddddd(m.vvv)), + m.fffff( + m.rrr('mmmmmmmmmmmmmmmm', 'sssssssssssssssssssssss'), + dict( + ffffffffffffffff, **{ + 'mmmmmm:ssssss': + m.rrrrrrrrrrr('|'.join(iiiiiiiiiiiiii), iiiiii=True) + })) + | m.wwwwww(m.rrrr('1h')) + | m.ggggggg(bbbbbbbbbbbbbbb)) + | m.jjjj() + | m.ppppp(m.vvv[0] + m.vvv[1])) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20127686(self): + code = textwrap.dedent("""\ + def f(): + if True: + return ((m.fffff( + m.rrr('xxxxxxxxxxxxxxxx', + 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy'), + mmmmmmmm) + | m.wwwwww(m.rrrr(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) + | m.ggggggg(self.gggggggg, m.sss()), m.fffff('aaaaaaaaaaaaaaaa') + | m.wwwwww(m.ddddd(self.tttttttttt, self.mmmmmmmmmmmmmmmmmmmmm)) + | m.ggggggg(self.gggggggg)) + | m.jjjj() + | m.ppppp(m.VAL[0] / m.VAL[1])) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20016122(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_penalty_import_names: 35}')) + unformatted_code = textwrap.dedent("""\ + from a_very_long_or_indented_module_name_yada_yada import (long_argument_1, + long_argument_2) + """) + expected_formatted_code = textwrap.dedent("""\ + from a_very_long_or_indented_module_name_yada_yada import ( + long_argument_1, long_argument_2) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: chromium, ' + 'split_before_logical_operator: True}')) + code = textwrap.dedent("""\ + class foo(): + + def __eq__(self, other): + return (isinstance(other, type(self)) + and self.xxxxxxxxxxx == other.xxxxxxxxxxx + and self.xxxxxxxx == other.xxxxxxxx + and self.aaaaaaaaaaaa == other.aaaaaaaaaaaa + and self.bbbbbbbbbbb == other.bbbbbbbbbbb + and self.ccccccccccccccccc == other.ccccccccccccccccc + and self.ddddddddddddddddddddddd == other.ddddddddddddddddddddddd + and self.eeeeeeeeeeee == other.eeeeeeeeeeee + and self.ffffffffffffff == other.time_completed + and self.gggggg == other.gggggg and self.hhh == other.hhh + and len(self.iiiiiiii) == len(other.iiiiiiii) + and all(jjjjjjj in other.iiiiiiii for jjjjjjj in self.iiiiiiii)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testB22527411(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee(ffffffffffffff) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + aaaaaa.bbbbbbbbbbbbbbbbbbbb[-1].cccccccccccccc.ddd().eeeeeeee( + ffffffffffffff) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB20849933(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + aaaaaaaa = { + 'xxx': '%s/cccccc/ddddddddddddddddddd.jar' % + (eeeeee.FFFFFFFFFFFFFFFFFF), + } + """) + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + if True: + aaaaaaaa = { + 'xxx': + '%s/cccccc/ddddddddddddddddddd.jar' % (eeeeee.FFFFFFFFFFFFFFFFFF), + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB20813997(self): + code = textwrap.dedent("""\ + def myfunc_1(): + myarray = numpy.zeros((2, 2, 2)) + print(myarray[:, 1, :]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20605036(self): + code = textwrap.dedent("""\ + foo = { + 'aaaa': { + # A comment for no particular reason. + 'xxxxxxxx': 'bbbbbbbbb', + 'yyyyyyyyyyyyyyyyyy': 'cccccccccccccccccccccccccccccc' + 'dddddddddddddddddddddddddddddddddddddddddd', + } + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20562732(self): + code = textwrap.dedent("""\ + foo = [ + # Comment about first list item + 'First item', + # Comment about second list item + 'Second item', + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20128830(self): + code = textwrap.dedent("""\ + a = { + 'xxxxxxxxxxxxxxxxxxxx': { + 'aaaa': + 'mmmmmmm', + 'bbbbb': + 'mmmmmmmmmmmmmmmmmmmmm', + 'cccccccccc': [ + 'nnnnnnnnnnn', + 'ooooooooooo', + 'ppppppppppp', + 'qqqqqqqqqqq', + ], + }, + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB20073838(self): + code = textwrap.dedent("""\ + class DummyModel(object): + + def do_nothing(self, class_1_count): + if True: + class_0_count = num_votes - class_1_count + return ('{class_0_name}={class_0_count}, {class_1_name}={class_1_count}' + .format( + class_0_name=self.class_0_name, + class_0_count=class_0_count, + class_1_name=self.class_1_name, + class_1_count=class_1_count)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19626808(self): + code = textwrap.dedent("""\ + if True: + aaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbb( + 'ccccccccccc', ddddddddd='eeeee').fffffffff([ggggggggggggggggggggg]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19547210(self): + code = textwrap.dedent("""\ + while True: + if True: + if True: + if True: + if xxxxxxxxxxxx.yyyyyyy(aa).zzzzzzz() not in ( + xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz, + xxxxxxxxxxxx.yyyyyyyyyyyyyy.zzzzzzzz): + continue + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19377034(self): + code = textwrap.dedent("""\ + def f(): + if (aaaaaaaaaaaaaaa.start >= aaaaaaaaaaaaaaa.end or + bbbbbbbbbbbbbbb.start >= bbbbbbbbbbbbbbb.end): + return False + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19372573(self): + code = textwrap.dedent("""\ + def f(): + if a: return 42 + while True: + if b: continue + if c: break + return 0 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + try: + style.SetGlobalStyle(style.CreatePEP8Style()) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreateChromiumStyle()) + + def testB19353268(self): + code = textwrap.dedent("""\ + a = {1, 2, 3}[x] + b = {'foo': 42, 'bar': 37}['foo'] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19287512(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + with xxxxxxxxxx.yyyyy( + 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', + fffffffffff=(aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd + .Mmmmmmmmmmmmmmmmmm(-1, 'permission error'))): + self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + with xxxxxxxxxx.yyyyy( + 'aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.eeeeeeeeeee', + fffffffffff=( + aaaaaaa.bbbbbbbb.ccccccc.dddddddddddddddddddd.Mmmmmmmmmmmmmmmmmm( + -1, 'permission error'))): + self.assertRaises(nnnnnnnnnnnnnnnn.ooooo, ppppp.qqqqqqqqqqqqqqqqq) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB19194420(self): + code = textwrap.dedent("""\ + method.Set( + 'long argument goes here that causes the line to break', + lambda arg2=0.5: arg2) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB19073499(self): + code = """\ +instance = ( + aaaaaaa.bbbbbbb().ccccccccccccccccc().ddddddddddd({ + 'aa': 'context!' + }).eeeeeeeeeeeeeeeeeee({ # Inline comment about why fnord has the value 6. + 'fnord': 6 + })) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB18257115(self): + code = textwrap.dedent("""\ + if True: + if True: + self._Test(aaaa, bbbbbbb.cccccccccc, dddddddd, eeeeeeeeeee, + [ffff, ggggggggggg, hhhhhhhhhhhh, iiiiii, jjjj]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB18256666(self): + code = textwrap.dedent("""\ + class Foo(object): + + def Bar(self): + aaaaa.bbbbbbb( + ccc='ddddddddddddddd', + eeee='ffffffffffffffffffffff-%s-%s' % (gggg, int(time.time())), + hhhhhh={ + 'iiiiiiiiiii': iiiiiiiiiii, + 'jjjj': jjjj.jjjjj(), + 'kkkkkkkkkkkk': kkkkkkkkkkkk, + }, + llllllllll=mmmmmm.nnnnnnnnnnnnnnnn) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB18256826(self): + code = textwrap.dedent("""\ + if True: + pass + # A multiline comment. + # Line two. + elif False: + pass + + if True: + pass + # A multiline comment. + # Line two. + elif False: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB18255697(self): + code = textwrap.dedent("""\ + AAAAAAAAAAAAAAA = { + 'XXXXXXXXXXXXXX': 4242, # Inline comment + # Next comment + 'YYYYYYYYYYYYYYYY': ['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'], + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testB17534869(self): + unformatted_code = textwrap.dedent("""\ + if True: + self.assertLess(abs(time.time()-aaaa.bbbbbbbbbbb( + datetime.datetime.now())), 1) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + self.assertLess( + abs(time.time() - aaaa.bbbbbbbbbbb(datetime.datetime.now())), 1) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB17489866(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + if True: + return aaaa.bbbbbbbbb(ccccccc=dddddddddddddd({('eeee', \ +'ffffffff'): str(j)})) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + if True: + return aaaa.bbbbbbbbb( + ccccccc=dddddddddddddd({ + ('eeee', 'ffffffff'): str(j) + })) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB17133019(self): + unformatted_code = textwrap.dedent("""\ + class aaaaaaaaaaaaaa(object): + + def bbbbbbbbbb(self): + with io.open("/dev/null", "rb"): + with io.open(os.path.join(aaaaa.bbbbb.ccccccccccc, + DDDDDDDDDDDDDDD, + "eeeeeeeee ffffffffff" + ), "rb") as gggggggggggggggggggg: + print(gggggggggggggggggggg) + """) + expected_formatted_code = textwrap.dedent("""\ + class aaaaaaaaaaaaaa(object): + + def bbbbbbbbbb(self): + with io.open("/dev/null", "rb"): + with io.open( + os.path.join(aaaaa.bbbbb.ccccccccccc, DDDDDDDDDDDDDDD, + "eeeeeeeee ffffffffff"), "rb") as gggggggggggggggggggg: + print(gggggggggggggggggggg) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB17011869(self): + unformatted_code = textwrap.dedent("""\ + '''blah......''' + + class SomeClass(object): + '''blah.''' + + AAAAAAAAAAAA = { # Comment. + 'BBB': 1.0, + 'DDDDDDDD': 0.4811 + } + """) + expected_formatted_code = textwrap.dedent("""\ + '''blah......''' + + + class SomeClass(object): + '''blah.''' + + AAAAAAAAAAAA = { # Comment. + 'BBB': 1.0, + 'DDDDDDDD': 0.4811 + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB16783631(self): + unformatted_code = textwrap.dedent("""\ + if True: + with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc(ddddddddddddd, + eeeeeeeee=self.fffffffffffff + )as gggg: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + with aaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccc( + ddddddddddddd, eeeeeeeee=self.fffffffffffff) as gggg: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB16572361(self): + unformatted_code = textwrap.dedent("""\ + def foo(self): + def bar(my_dict_name): + self.my_dict_name['foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with('foo_bar_baz_boo') + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(self): + + def bar(my_dict_name): + self.my_dict_name[ + 'foo-bar-baz-biz-boo-baa-baa'].IncrementBy.assert_called_once_with( + 'foo_bar_baz_boo') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB15884241(self): + unformatted_code = textwrap.dedent("""\ + if 1: + if 1: + for row in AAAA: + self.create(aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % row [0].replace(".foo", ".bar"), aaaaa=bbb[1], ccccc=bbb[2], dddd=bbb[3], eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], ffffffff=[s.strip() for s in bbb[5].split(",")], gggggg=bbb[6]) + """) + expected_formatted_code = textwrap.dedent("""\ + if 1: + if 1: + for row in AAAA: + self.create( + aaaaaaaa="/aaa/bbbb/cccc/dddddd/eeeeeeeeeeeeeeeeeeeeeeeeee/%s" % + row[0].replace(".foo", ".bar"), + aaaaa=bbb[1], + ccccc=bbb[2], + dddd=bbb[3], + eeeeeeeeeee=[s.strip() for s in bbb[4].split(",")], + ffffffff=[s.strip() for s in bbb[5].split(",")], + gggggg=bbb[6]) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB15697268(self): + unformatted_code = textwrap.dedent("""\ + def main(unused_argv): + ARBITRARY_CONSTANT_A = 10 + an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) + ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = map(math.sqrt, an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) + a_long_name_slicing = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = ("I am a crazy, no good, string whats too long, etc." + " no really ")[:ARBITRARY_CONSTANT_A] + """) + expected_formatted_code = textwrap.dedent("""\ + def main(unused_argv): + ARBITRARY_CONSTANT_A = 10 + an_array_with_an_exceedingly_long_name = range(ARBITRARY_CONSTANT_A + 1) + ok = an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A] + bad_slice = map(math.sqrt, + an_array_with_an_exceedingly_long_name[:ARBITRARY_CONSTANT_A]) + a_long_name_slicing = an_array_with_an_exceedingly_long_name[: + ARBITRARY_CONSTANT_A] + bad_slice = ("I am a crazy, no good, string whats too long, etc." + + " no really ")[:ARBITRARY_CONSTANT_A] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB15597568(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + print(("Return code was %d" + (", and the process timed out." if did_time_out else ".")) % errorcode) + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + print(("Return code was %d" + (", and the process timed out." + if did_time_out else ".")) % errorcode) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB15542157(self): + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, gggggg.hhhhhhhhhhhhhhhhh) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaa = bbbb.ccccccccccccccc(dddddd.eeeeeeeeeeeeee, ffffffffffffffffff, + gggggg.hhhhhhhhhhhhhhhhh) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB15438132(self): + unformatted_code = textwrap.dedent("""\ + if aaaaaaa.bbbbbbbbbb: + cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) + if hhhhhh.iiiii.jjjjjjjjjjjjj: + # This is a comment in the middle of it all. + kkkkkkk.llllllllll.mmmmmmmmmmmmm = True + if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or + eeeeee.fffff.ggggggggggggggggggggggggggg() != hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): + aaaaaaaa.bbbbbbbbbbbb( + aaaaaa.bbbbb.cc, + dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( + gggggg.hh, + iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, + lllll.mm), + nnnnnnnnnn=ooooooo.pppppppppp) + """) + expected_formatted_code = textwrap.dedent("""\ + if aaaaaaa.bbbbbbbbbb: + cccccc.dddddddddd(eeeeeeeeeee=fffffffffffff.gggggggggggggggggg) + if hhhhhh.iiiii.jjjjjjjjjjjjj: + # This is a comment in the middle of it all. + kkkkkkk.llllllllll.mmmmmmmmmmmmm = True + if (aaaaaa.bbbbb.ccccccccccccc != ddddddd.eeeeeeeeee.fffffffffffff or + eeeeee.fffff.ggggggggggggggggggggggggggg() != + hhhhhhh.iiiiiiiiii.jjjjjjjjjjjj): + aaaaaaaa.bbbbbbbbbbbb( + aaaaaa.bbbbb.cc, + dddddddddddd=eeeeeeeeeeeeeeeeeee.fffffffffffffffff( + gggggg.hh, iiiiiiiiiiiiiiiiiii.jjjjjjjjjj.kkkkkkk, lllll.mm), + nnnnnnnnnn=ooooooo.pppppppppp) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB14468247(self): + unformatted_code = """\ +call(a=1, + b=2, +) +""" + expected_formatted_code = """\ +call( + a=1, + b=2, +) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB14406499(self): + unformatted_code = textwrap.dedent("""\ + def foo1(parameter_1, parameter_2, parameter_3, parameter_4, \ +parameter_5, parameter_6): pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, + parameter_6): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB13900309(self): + unformatted_code = textwrap.dedent("""\ + self.aaaaaaaaaaa( # A comment in the middle of it all. + 948.0/3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) + """) + expected_formatted_code = textwrap.dedent("""\ + self.aaaaaaaaaaa( # A comment in the middle of it all. + 948.0 / 3600, self.bbb.ccccccccccccccccccccc(dddddddddddddddd.eeee, True)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + code = textwrap.dedent("""\ + aaaaaaaaaa.bbbbbbbbbbbbbbbbbbbbbbbb.cccccccccccccccccccccccccccccc( + DC_1, (CL - 50, CL), AAAAAAAA, BBBBBBBBBBBBBBBB, 98.0, + CCCCCCC).ddddddddd( # Look! A comment is here. + AAAAAAAA - (20 * 60 - 5)) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc().dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( + ).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc(x).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa.bbbbbbbbbbbbb.ccccccccccccccccccccccccc( + x).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa(xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa( + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx).dddddddddddddddddddddddddd(1, 2, 3, 4) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc().\ +dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff().gggggggggggggggggg() + """) + expected_formatted_code = textwrap.dedent("""\ + aaaaaaaaaaaaaaaaaaaaaaaa().bbbbbbbbbbbbbbbbbbbbbbbb().ccccccccccccccccccc( + ).dddddddddddddddddd().eeeeeeeeeeeeeeeeeeeee().fffffffffffffffff( + ).gggggggggggggggggg() + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testB67935687(self): + code = textwrap.dedent("""\ + Fetch( + Raw('monarch.BorgTask', '/union/row_operator_action_delay'), + {'borg_user': self.borg_user}) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + shelf_renderer.expand_text = text.translate_to_unicode( + expand_text % { + 'creator': creator + }) + """) + expected_formatted_code = textwrap.dedent("""\ + shelf_renderer.expand_text = text.translate_to_unicode( + expand_text % {'creator': creator}) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_facebook_test.py b/yapftests/reformatter_facebook_test.py new file mode 100644 index 0000000..289aa85 --- /dev/null +++ b/yapftests/reformatter_facebook_test.py @@ -0,0 +1,432 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Facebook tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForFacebookStyle(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreateFacebookStyle()) + + def testNoNeedForLineBreaks(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + just_one_arg, **kwargs): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name(just_one_arg, **kwargs): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDedentClosingBracket(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, second_argument_makes_the_line_too_long + ): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBreakAfterOpeningBracketIfContentsTooBig(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name(a, b, c, d, e, f, g, h, i, j, k, l, m, + n, o, p, q, r, s, t, u, v, w, x, y, z): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, \ +v, w, x, y, z + ): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDedentClosingBracketWithComments(self): + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + # comment about the first argument + first_argument_with_a_very_long_name_or_so, + # comment about the second argument + second_argument_makes_the_line_too_long): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name( + # comment about the first argument + first_argument_with_a_very_long_name_or_so, + # comment about the second argument + second_argument_makes_the_line_too_long + ): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDedentImportAsNames(self): + code = textwrap.dedent("""\ + from module import ( + internal_function as function, + SOME_CONSTANT_NUMBER1, + SOME_CONSTANT_NUMBER2, + SOME_CONSTANT_NUMBER3, + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDedentTestListGexp(self): + unformatted_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError, + ) as exception: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + + try: + pass + except ( + IOError, + OSError, + LookupError, + RuntimeError, + OverflowError, + ) as exception: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testBrokenIdempotency(self): + # TODO(ambv): The following behaviour should be fixed. + pass0_code = textwrap.dedent("""\ + try: + pass + except (IOError, OSError, LookupError, RuntimeError, OverflowError) as exception: + pass + """) + pass1_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(pass0_code) + self.assertCodeEqual(pass1_code, reformatter.Reformat(uwlines)) + + pass2_code = textwrap.dedent("""\ + try: + pass + except ( + IOError, OSError, LookupError, RuntimeError, OverflowError + ) as exception: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(pass1_code) + self.assertCodeEqual(pass2_code, reformatter.Reformat(uwlines)) + + def testIfExprHangingIndent(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1 + ): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSimpleDedenting(self): + unformatted_code = textwrap.dedent("""\ + if True: + self.assertEqual(result.reason_not_added, "current preflight is still running") + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + self.assertEqual( + result.reason_not_added, "current preflight is still running" + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDedentingWithSubscripts(self): + unformatted_code = textwrap.dedent("""\ + class Foo: + class Bar: + @classmethod + def baz(cls, clues_list, effect, constraints, constraint_manager): + if clues_lists: + return cls.single_constraint_not(clues_lists, effect, constraints[0], constraint_manager) + + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo: + class Bar: + @classmethod + def baz(cls, clues_list, effect, constraints, constraint_manager): + if clues_lists: + return cls.single_constraint_not( + clues_lists, effect, constraints[0], constraint_manager + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testDedentingCallsWithInnerLists(self): + code = textwrap.dedent("""\ + class _(): + def _(): + cls.effect_clues = { + 'effect': Clue((cls.effect_time, 'apache_host'), effect_line, 40) + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDedentingListComprehension(self): + unformatted_code = textwrap.dedent("""\ + class Foo(): + def _pack_results_for_constraint_or(): + self.param_groups = dict( + ( + key + 1, ParamGroup(groups[key], default_converter) + ) for key in six.moves.range(len(groups)) + ) + + for combination in cls._clues_combinations(clues_lists): + if all( + cls._verify_constraint(combination, effect, constraint) + for constraint in constraints + ): + pass + + guessed_dict = dict( + ( + key, guessed_pattern_matches[key] + ) for key in six.moves.range(len(guessed_pattern_matches)) + ) + + content = "".join( + itertools.chain( + (first_line_fragment, ), lines_between, (last_line_fragment, ) + ) + ) + + rule = Rule( + [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, constraints1, + Rule.LINKAGE_AND + ) + + assert sorted(log_type.files_to_parse) == [ + ('localhost', os.path.join(path, 'node_1.log'), super_parser), + ('localhost', os.path.join(path, 'node_2.log'), super_parser) + ] + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(): + def _pack_results_for_constraint_or(): + self.param_groups = dict( + (key + 1, ParamGroup(groups[key], default_converter)) + for key in six.moves.range(len(groups)) + ) + + for combination in cls._clues_combinations(clues_lists): + if all( + cls._verify_constraint(combination, effect, constraint) + for constraint in constraints + ): + pass + + guessed_dict = dict( + (key, guessed_pattern_matches[key]) + for key in six.moves.range(len(guessed_pattern_matches)) + ) + + content = "".join( + itertools.chain( + (first_line_fragment, ), lines_between, (last_line_fragment, ) + ) + ) + + rule = Rule( + [self.cause1, self.cause2, self.cause1, self.cause2], self.effect, + constraints1, Rule.LINKAGE_AND + ) + + assert sorted(log_type.files_to_parse) == [ + ('localhost', os.path.join(path, 'node_1.log'), super_parser), + ('localhost', os.path.join(path, 'node_2.log'), super_parser) + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMustSplitDedenting(self): + code = textwrap.dedent("""\ + class _(): + def _(): + effect_line = FrontInput( + effect_line_offset, line_content, + LineSource('localhost', xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx) + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDedentIfConditional(self): + code = textwrap.dedent("""\ + class _(): + def _(): + if True: + if not self.frobbies and ( + self.foobars.counters['db.cheeses'] != 1 or + self.foobars.counters['db.marshmellow_skins'] != 1 + ): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDedentSet(self): + code = textwrap.dedent("""\ + class _(): + def _(): + assert set(self.constraint_links.get_links()) == set( + [ + (2, 10, 100), + (2, 10, 200), + (2, 20, 100), + (2, 20, 200), + ] + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testDedentingInnerScope(self): + code = textwrap.dedent("""\ + class Foo(): + @classmethod + def _pack_results_for_constraint_or(cls, combination, constraints): + return cls._create_investigation_result( + (clue for clue in combination if not clue == Verifier.UNMATCHED), + constraints, InvestigationResult.OR + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + + uwlines = yapf_test_helper.ParseAndUnwrap(reformatted_code) + reformatted_code = reformatter.Reformat(uwlines) + self.assertCodeEqual(code, reformatted_code) + + def testCommentWithNewlinesInPrefix(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + if 0: + return False + + + #a deadly comment + elif 1: + return True + + + print(foo()) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + if 0: + return False + + #a deadly comment + elif 1: + return True + + + print(foo()) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testIfStmtClosingBracket(self): + unformatted_code = """\ +if (isinstance(value , (StopIteration , StopAsyncIteration )) and exc.__cause__ is value_asdfasdfasdfasdfsafsafsafdasfasdfs): + return False +""" + expected_formatted_code = """\ +if ( + isinstance(value, (StopIteration, StopAsyncIteration)) and + exc.__cause__ is value_asdfasdfasdfasdfsafsafsafdasfasdfs +): + return False +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_pep8_test.py b/yapftests/reformatter_pep8_test.py new file mode 100644 index 0000000..a6f736f --- /dev/null +++ b/yapftests/reformatter_pep8_test.py @@ -0,0 +1,436 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PEP8 tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForPEP8Style(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testIndent4(self): + unformatted_code = textwrap.dedent("""\ + if a+b: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if a + b: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSingleLineIfStatements(self): + code = textwrap.dedent("""\ + if True: a = 42 + elif False: b = 42 + else: c = 42 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testNoBlankBetweenClassAndDef(self): + unformatted_code = textwrap.dedent("""\ + class Foo: + + def joe(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo: + def joe(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSingleWhiteBeforeTrailingComment(self): + unformatted_code = textwrap.dedent("""\ + if a+b: # comment + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if a + b: # comment + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSpaceBetweenEndingCommandAndClosingBracket(self): + unformatted_code = textwrap.dedent("""\ + a = ( + 1, + ) + """) + expected_formatted_code = textwrap.dedent("""\ + a = (1, ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testContinuedNonOutdentedLine(self): + code = textwrap.dedent("""\ + class eld(d): + if str(geom.geom_type).upper( + ) != self.geom_type and not self.geom_type == 'GEOMETRY': + ror(code='om_type') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testWrappingPercentExpressions(self): + unformatted_code = textwrap.dedent("""\ + def f(): + if True: + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, xxxxxxxxxxxxxxxxxxxxx + 1) + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + if True: + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxx.yyy + 1) + zzzzz = '%s-%s' % (xxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxxxxxx + 1) + zzzzz = '%s-%s'.ww(xxxxxxxxxxxxxxxxxxxxxxx + 1, + xxxxxxxxxxxxxxxxxxxxx + 1) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testAlignClosingBracketWithVisualIndentation(self): + unformatted_code = textwrap.dedent("""\ + TEST_LIST = ('foo', 'bar', # first comment + 'baz' # second comment + ) + """) + expected_formatted_code = textwrap.dedent("""\ + TEST_LIST = ( + 'foo', + 'bar', # first comment + 'baz' # second comment + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + def f(): + + def g(): + while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb' + ): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def f(): + def g(): + while (xxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' + and xxxxxxxxxxxxxxxxxxxx( + yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == 'bbbbbbb'): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testIndentSizeChanging(self): + unformatted_code = textwrap.dedent("""\ + if True: + runtime_mins = (program_end_time - program_start_time).total_seconds() / 60.0 + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + runtime_mins = ( + program_end_time - program_start_time).total_seconds() / 60.0 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testHangingIndentCollision(self): + unformatted_code = textwrap.dedent("""\ + if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx or yyyyyyyyyyyyyyyyy): + pass + elif (xxxxxxxxxxxxxxx(aaaaaaaaaaa, bbbbbbbbbbbbbb, cccccccccccc, dddddddddd=None)): + pass + + + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + for connection in itertools.chain(branch.contact, branch.address, morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): + dosomething(connection) + """) + expected_formatted_code = textwrap.dedent("""\ + if (aaaaaaaaaaaaaa + bbbbbbbbbbbbbbbb == ccccccccccccccccc and xxxxxxxxxxxxx + or yyyyyyyyyyyyyyyyy): + pass + elif (xxxxxxxxxxxxxxx( + aaaaaaaaaaa, bbbbbbbbbbbbbb, cccccccccccc, dddddddddd=None)): + pass + + + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + for connection in itertools.chain( + branch.contact, branch.address, + morestuff.andmore.andmore.andmore.andmore.andmore.andmore.andmore): + dosomething(connection) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingBeforeLogicalOperator(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_before_logical_operator: True}')) + unformatted_code = textwrap.dedent("""\ + def foo(): + return bool(update.message.new_chat_member or update.message.left_chat_member or + update.message.new_chat_title or update.message.new_chat_photo or + update.message.delete_chat_photo or update.message.group_chat_created or + update.message.supergroup_chat_created or update.message.channel_chat_created + or update.message.migrate_to_chat_id or update.message.migrate_from_chat_id or + update.message.pinned_message) + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + return bool( + update.message.new_chat_member or update.message.left_chat_member + or update.message.new_chat_title or update.message.new_chat_photo + or update.message.delete_chat_photo + or update.message.group_chat_created + or update.message.supergroup_chat_created + or update.message.channel_chat_created + or update.message.migrate_to_chat_id + or update.message.migrate_from_chat_id + or update.message.pinned_message) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testContiguousListEndingWithComment(self): + unformatted_code = textwrap.dedent("""\ + if True: + if True: + keys.append(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. + """) + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + keys.append( + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) # may be unassigned. + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingBeforeFirstArgument(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, split_before_first_argument: True}')) + unformatted_code = textwrap.dedent("""\ + a_very_long_function_name(long_argument_name_1=1, long_argument_name_2=2, + long_argument_name_3=3, long_argument_name_4=4) + """) + expected_formatted_code = textwrap.dedent("""\ + a_very_long_function_name( + long_argument_name_1=1, + long_argument_name_2=2, + long_argument_name_3=3, + long_argument_name_4=4) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testSplittingExpressionsInsideSubscripts(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + df = df[(df['campaign_status'] == 'LIVE') & (df['action_status'] == 'LIVE')] + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + df = df[(df['campaign_status'] == 'LIVE') + & (df['action_status'] == 'LIVE')] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplitListsAndDictSetMakersIfCommaTerminated(self): + unformatted_code = textwrap.dedent("""\ + DJANGO_TEMPLATES_OPTIONS = {"context_processors": []} + DJANGO_TEMPLATES_OPTIONS = {"context_processors": [],} + x = ["context_processors"] + x = ["context_processors",] + """) + expected_formatted_code = textwrap.dedent("""\ + DJANGO_TEMPLATES_OPTIONS = {"context_processors": []} + DJANGO_TEMPLATES_OPTIONS = { + "context_processors": [], + } + x = ["context_processors"] + x = [ + "context_processors", + ] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplitAroundNamedAssigns(self): + unformatted_code = textwrap.dedent("""\ + class a(): + def a(): return a( + aaaaaaaaaa=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) + """) + expected_formatted_code = textwrap.dedent("""\ + class a(): + def a(): + return a( + aaaaaaaaaa=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testUnaryOperator(self): + unformatted_code = textwrap.dedent("""\ + if not -3 < x < 3: + pass + if -3 < x < 3: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + if not -3 < x < 3: + pass + if -3 < x < 3: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoSplitBeforeDictValue(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{based_on_style: pep8, ' + 'allow_split_before_dict_value: false, ' + 'coalesce_brackets: true, ' + 'dedent_closing_brackets: true, ' + 'each_dict_entry_on_separate_line: true, ' + 'split_before_logical_operator: true}')) + + unformatted_code = textwrap.dedent("""\ + some_dict = { + 'title': _("I am example data"), + 'description': _("Lorem ipsum dolor met sit amet elit, si vis pacem para bellum " + "elites nihi very long string."), + } + """) + expected_formatted_code = textwrap.dedent("""\ + some_dict = { + 'title': _("I am example data"), + 'description': _( + "Lorem ipsum dolor met sit amet elit, si vis pacem para bellum " + "elites nihi very long string." + ), + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + X = {'a': 1, 'b': 2, 'key': this_is_a_function_call_that_goes_over_the_column_limit_im_pretty_sure()} + """) + expected_formatted_code = textwrap.dedent("""\ + X = { + 'a': 1, + 'b': 2, + 'key': this_is_a_function_call_that_goes_over_the_column_limit_im_pretty_sure() + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + attrs = { + 'category': category, + 'role': forms.ModelChoiceField(label=_("Role"), required=False, queryset=category_roles, initial=selected_role, empty_label=_("No access"),), + } + """) + expected_formatted_code = textwrap.dedent("""\ + attrs = { + 'category': category, + 'role': forms.ModelChoiceField( + label=_("Role"), + required=False, + queryset=category_roles, + initial=selected_role, + empty_label=_("No access"), + ), + } + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + + unformatted_code = textwrap.dedent("""\ + css_class = forms.CharField( + label=_("CSS class"), + required=False, + help_text=_("Optional CSS class used to customize this category appearance from templates."), + ) + """) + expected_formatted_code = textwrap.dedent("""\ + css_class = forms.CharField( + label=_("CSS class"), + required=False, + help_text=_( + "Optional CSS class used to customize this category appearance from templates." + ), + ) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_python3_test.py b/yapftests/reformatter_python3_test.py new file mode 100644 index 0000000..9be6528 --- /dev/null +++ b/yapftests/reformatter_python3_test.py @@ -0,0 +1,383 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python 3 tests for yapf.reformatter.""" + +import sys +import textwrap +import unittest + +from yapf.yapflib import py3compat +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +@unittest.skipUnless(py3compat.PY3, 'Requires Python 3') +class TestsForPython3Code(yapf_test_helper.YAPFTest): + """Test a few constructs that are new Python 3 syntax.""" + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testTypedNames(self): + unformatted_code = textwrap.dedent("""\ + def x(aaaaaaaaaaaaaaa:int,bbbbbbbbbbbbbbbb:str,ccccccccccccccc:dict,eeeeeeeeeeeeee:set={1, 2, 3})->bool: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def x(aaaaaaaaaaaaaaa: int, + bbbbbbbbbbbbbbbb: str, + ccccccccccccccc: dict, + eeeeeeeeeeeeee: set = {1, 2, 3}) -> bool: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testKeywordOnlyArgSpecifier(self): + unformatted_code = textwrap.dedent("""\ + def foo(a, *, kw): + return a+kw + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(a, *, kw): + return a + kw + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + @unittest.skipUnless(py3compat.PY36, 'Requires Python 3.6') + def testPEP448ParameterExpansion(self): + unformatted_code = textwrap.dedent("""\ + { ** x } + { **{} } + { **{ **x }, **x } + {'a': 1, **kw , 'b':3, **kw2 } + """) + expected_formatted_code = textwrap.dedent("""\ + {**x} + {**{}} + {**{**x}, **x} + {'a': 1, **kw, 'b': 3, **kw2} + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testAnnotations(self): + unformatted_code = textwrap.dedent("""\ + def foo(a: list, b: "bar") -> dict: + return a+b + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(a: list, b: "bar") -> dict: + return a + b + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testExecAsNonKeyword(self): + unformatted_code = 'methods.exec( sys.modules[name])\n' + expected_formatted_code = 'methods.exec(sys.modules[name])\n' + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testAsyncFunctions(self): + if sys.version_info[1] < 5: + return + code = textwrap.dedent("""\ + import asyncio + import time + + + @print_args + async def slow_operation(): + await asyncio.sleep(1) + # print("Slow operation {} complete".format(n)) + + + async def main(): + start = time.time() + if (await get_html()): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines, verify=False)) + + def testNoSpacesAroundPowerOperator(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, SPACES_AROUND_POWER_OPERATOR: True}')) + unformatted_code = textwrap.dedent("""\ + a**b + """) + expected_formatted_code = textwrap.dedent("""\ + a ** b + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testSpacesAroundDefaultOrNamedAssign(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'SPACES_AROUND_DEFAULT_OR_NAMED_ASSIGN: True}')) + unformatted_code = textwrap.dedent("""\ + f(a=5) + """) + expected_formatted_code = textwrap.dedent("""\ + f(a = 5) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testTypeHint(self): + unformatted_code = textwrap.dedent("""\ + def foo(x: int=42): + pass + + + def foo2(x: 'int' =42): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(x: int = 42): + pass + + + def foo2(x: 'int' = 42): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMatrixMultiplication(self): + unformatted_code = textwrap.dedent("""\ + a=b@c + """) + expected_formatted_code = textwrap.dedent("""\ + a = b @ c + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testNoneKeyword(self): + code = """\ +None.__ne__() +""" + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testAsyncWithPrecedingComment(self): + if sys.version_info[1] < 5: + return + unformatted_code = textwrap.dedent("""\ + import asyncio + + # Comment + async def bar(): + pass + + async def foo(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + import asyncio + + + # Comment + async def bar(): + pass + + + async def foo(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testAsyncFunctionsNested(self): + if sys.version_info[1] < 5: + return + code = textwrap.dedent("""\ + async def outer(): + async def inner(): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testKeepTypesIntact(self): + if sys.version_info[1] < 5: + return + unformatted_code = textwrap.dedent("""\ + def _ReduceAbstractContainers( + self, *args: Optional[automation_converter.PyiCollectionAbc]) -> List[ + automation_converter.PyiCollectionAbc]: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def _ReduceAbstractContainers( + self, *args: Optional[automation_converter.PyiCollectionAbc] + ) -> List[automation_converter.PyiCollectionAbc]: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testContinuationIndentWithAsync(self): + if sys.version_info[1] < 5: + return + unformatted_code = textwrap.dedent("""\ + async def start_websocket(): + async with session.ws_connect( + r"ws://a_really_long_long_long_long_long_long_url") as ws: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + async def start_websocket(): + async with session.ws_connect( + r"ws://a_really_long_long_long_long_long_long_url") as ws: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testSplittingArguments(self): + if sys.version_info[1] < 5: + return + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig( + '{based_on_style: pep8, ' + 'dedent_closing_brackets: true, ' + 'coalesce_brackets: false, ' + 'space_between_ending_comma_and_closing_bracket: false, ' + 'split_arguments_when_comma_terminated: true, ' + 'split_before_first_argument: true}')) + unformatted_code = """\ +async def open_file(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): + pass + +async def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass + +def open_file(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): + pass + +def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass +""" + expected_formatted_code = """\ +async def open_file( + file, + mode='r', + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None +): + pass + + +async def run_sync_in_worker_thread( + sync_fn, *args, cancellable=False, limiter=None +): + pass + + +def open_file( + file, + mode='r', + buffering=-1, + encoding=None, + errors=None, + newline=None, + closefd=True, + opener=None +): + pass + + +def run_sync_in_worker_thread(sync_fn, *args, cancellable=False, limiter=None): + pass +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDictUnpacking(self): + if sys.version_info[1] < 5: + return + unformatted_code = """\ +class Foo: + def foo(self): + foofoofoofoofoofoofoofoo('foofoofoofoofoo', { + + 'foo': 'foo', + + **foofoofoo + }) +""" + expected_formatted_code = """\ +class Foo: + def foo(self): + foofoofoofoofoofoofoofoo('foofoofoofoofoo', { + 'foo': 'foo', + **foofoofoo + }) +""" + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testMultilineFormatString(self): + if sys.version_info[1] < 6: + return + code = """\ +# yapf: disable +(f''' + ''') +# yapf: enable +""" + # https://github.com/google/yapf/issues/513 + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + def testEllipses(self): + if sys.version_info[1] < 6: + return + code = """\ +def dirichlet(x12345678901234567890123456789012345678901234567890=...) -> None: + return +""" + # https://github.com/google/yapf/issues/533 + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self.assertCodeEqual(code, reformatter.Reformat(uwlines)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_style_config_test.py b/yapftests/reformatter_style_config_test.py new file mode 100644 index 0000000..5afd805 --- /dev/null +++ b/yapftests/reformatter_style_config_test.py @@ -0,0 +1,81 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Style config tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import reformatter +from yapf.yapflib import style + +from yapftests import yapf_test_helper + + +class TestsForStyleConfig(yapf_test_helper.YAPFTest): + + def setUp(self): + self.current_style = style.DEFAULT_STYLE + + def testSetGlobalStyle(self): + try: + style.SetGlobalStyle(style.CreateChromiumStyle()) + unformatted_code = textwrap.dedent(u"""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent(u"""\ + for i in range(5): + print('bar') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + unformatted_code = textwrap.dedent(u"""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent(u"""\ + for i in range(5): + print('bar') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(uwlines)) + + def testOperatorStyle(self): + try: + sympy_style = style.CreatePEP8Style() + sympy_style['NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS'] = \ + style._StringSetConverter('*,/') + style.SetGlobalStyle(sympy_style) + unformatted_code = textwrap.dedent("""\ + a = 1+2 * 3 - 4 / 5 + """) + expected_formatted_code = textwrap.dedent("""\ + a = 1 + 2*3 - 4/5 + """) + + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines)) + finally: + style.SetGlobalStyle(style.CreatePEP8Style()) + style.DEFAULT_STYLE = self.current_style + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/reformatter_verify_test.py b/yapftests/reformatter_verify_test.py new file mode 100644 index 0000000..1b3d5b0 --- /dev/null +++ b/yapftests/reformatter_verify_test.py @@ -0,0 +1,108 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.reformatter.""" + +import textwrap +import unittest + +from yapf.yapflib import py3compat +from yapf.yapflib import reformatter +from yapf.yapflib import style +from yapf.yapflib import verifier + +from yapftests import yapf_test_helper + + +@unittest.skipIf(py3compat.PY3, 'Requires Python 2') +class TestVerifyNoVerify(yapf_test_helper.YAPFTest): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testVerifyException(self): + unformatted_code = textwrap.dedent("""\ + class ABC(metaclass=type): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + with self.assertRaises(verifier.InternalError): + reformatter.Reformat(uwlines, verify=True) + reformatter.Reformat(uwlines) # verify should be False by default. + + def testNoVerify(self): + unformatted_code = textwrap.dedent("""\ + class ABC(metaclass=type): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class ABC(metaclass=type): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines, verify=False)) + + def testVerifyFutureImport(self): + unformatted_code = textwrap.dedent("""\ + from __future__ import print_function + + def call_my_function(the_function): + the_function("hi") + + if __name__ == "__main__": + call_my_function(print) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + with self.assertRaises(verifier.InternalError): + reformatter.Reformat(uwlines, verify=True) + + expected_formatted_code = textwrap.dedent("""\ + from __future__ import print_function + + + def call_my_function(the_function): + the_function("hi") + + + if __name__ == "__main__": + call_my_function(print) + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines, verify=False)) + + def testContinuationLineShouldBeDistinguished(self): + unformatted_code = textwrap.dedent("""\ + class Foo(object): + + def bar(self): + if self.solo_generator_that_is_long is None and len( + self.generators + self.next_batch) == 1: + pass + """) + expected_formatted_code = textwrap.dedent("""\ + class Foo(object): + def bar(self): + if self.solo_generator_that_is_long is None and len( + self.generators + self.next_batch) == 1: + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, + reformatter.Reformat(uwlines, verify=False)) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/split_penalty_test.py b/yapftests/split_penalty_test.py new file mode 100644 index 0000000..7d500da --- /dev/null +++ b/yapftests/split_penalty_test.py @@ -0,0 +1,259 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.split_penalty.""" + +import sys +import textwrap +import unittest + +from lib2to3 import pytree + +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import split_penalty + +UNBREAKABLE = split_penalty.UNBREAKABLE +VERY_STRONGLY_CONNECTED = split_penalty.VERY_STRONGLY_CONNECTED +DOTTED_NAME = split_penalty.DOTTED_NAME +STRONGLY_CONNECTED = split_penalty.STRONGLY_CONNECTED + + +class SplitPenaltyTest(unittest.TestCase): + + def _ParseAndComputePenalties(self, code, dumptree=False): + """Parses the code and computes split penalties. + + Arguments: + code: code to parse as a string + dumptree: if True, the parsed pytree (after penalty assignment) is dumped + to stderr. Useful for debugging. + + Returns: + Parse tree. + """ + tree = pytree_utils.ParseCodeToTree(code) + split_penalty.ComputeSplitPenalties(tree) + if dumptree: + pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) + return tree + + def _CheckPenalties(self, tree, list_of_expected): + """Check that the tokens in the tree have the correct penalties. + + Args: + tree: the pytree. + list_of_expected: list of (name, penalty) pairs. Non-semantic tokens are + filtered out from the expected values. + """ + + def FlattenRec(tree): + if pytree_utils.NodeName(tree) in pytree_utils.NONSEMANTIC_TOKENS: + return [] + if isinstance(tree, pytree.Leaf): + return [(tree.value, + pytree_utils.GetNodeAnnotation( + tree, pytree_utils.Annotation.SPLIT_PENALTY))] + nodes = [] + for node in tree.children: + nodes += FlattenRec(node) + return nodes + + self.assertEqual(list_of_expected, FlattenRec(tree)) + + def testUnbreakable(self): + # Test function definitions. + code = textwrap.dedent(r""" + def foo(x): + pass + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('def', None), + ('foo', UNBREAKABLE), + ('(', UNBREAKABLE), + ('x', None), + (')', STRONGLY_CONNECTED), + (':', UNBREAKABLE), + ('pass', None), + ]) + + # Test function definition with trailing comment. + code = textwrap.dedent(r""" + def foo(x): # trailing comment + pass + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('def', None), + ('foo', UNBREAKABLE), + ('(', UNBREAKABLE), + ('x', None), + (')', STRONGLY_CONNECTED), + (':', UNBREAKABLE), + ('pass', None), + ]) + + # Test class definitions. + code = textwrap.dedent(r""" + class A: + pass + class B(A): + pass + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('class', None), + ('A', UNBREAKABLE), + (':', UNBREAKABLE), + ('pass', None), + ('class', None), + ('B', UNBREAKABLE), + ('(', UNBREAKABLE), + ('A', None), + (')', None), + (':', UNBREAKABLE), + ('pass', None), + ]) + + # Test lambda definitions. + code = textwrap.dedent(r""" + lambda a, b: None + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('lambda', None), + ('a', UNBREAKABLE), + (',', UNBREAKABLE), + ('b', UNBREAKABLE), + (':', UNBREAKABLE), + ('None', UNBREAKABLE), + ]) + + # Test dotted names. + code = textwrap.dedent(r""" + import a.b.c + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('import', None), + ('a', None), + ('.', UNBREAKABLE), + ('b', UNBREAKABLE), + ('.', UNBREAKABLE), + ('c', UNBREAKABLE), + ]) + + def testStronglyConnected(self): + # Test dictionary keys. + code = textwrap.dedent(r""" + a = { + 'x': 42, + y(lambda a: 23): 37, + } + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('a', None), + ('=', None), + ('{', None), + ("'x'", None), + (':', STRONGLY_CONNECTED), + ('42', None), + (',', None), + ('y', None), + ('(', UNBREAKABLE), + ('lambda', STRONGLY_CONNECTED), + ('a', UNBREAKABLE), + (':', UNBREAKABLE), + ('23', UNBREAKABLE), + (')', VERY_STRONGLY_CONNECTED), + (':', STRONGLY_CONNECTED), + ('37', None), + (',', None), + ('}', None), + ]) + + # Test list comprehension. + code = textwrap.dedent(r""" + [a for a in foo if a.x == 37] + """) + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('[', None), + ('a', None), + ('for', 0), + ('a', STRONGLY_CONNECTED), + ('in', STRONGLY_CONNECTED), + ('foo', STRONGLY_CONNECTED), + ('if', 0), + ('a', STRONGLY_CONNECTED), + ('.', UNBREAKABLE), + ('x', DOTTED_NAME), + ('==', STRONGLY_CONNECTED), + ('37', STRONGLY_CONNECTED), + (']', None), + ]) + + def testFuncCalls(self): + code = 'foo(1, 2, 3)\n' + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('foo', None), + ('(', UNBREAKABLE), + ('1', None), + (',', UNBREAKABLE), + ('2', None), + (',', UNBREAKABLE), + ('3', None), + (')', VERY_STRONGLY_CONNECTED), + ]) + + # Now a method call, which has more than one trailer + code = 'foo.bar.baz(1, 2, 3)\n' + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('foo', None), + ('.', UNBREAKABLE), + ('bar', DOTTED_NAME), + ('.', STRONGLY_CONNECTED), + ('baz', DOTTED_NAME), + ('(', STRONGLY_CONNECTED), + ('1', None), + (',', UNBREAKABLE), + ('2', None), + (',', UNBREAKABLE), + ('3', None), + (')', VERY_STRONGLY_CONNECTED), + ]) + + # Test single generator argument. + code = 'max(i for i in xrange(10))\n' + tree = self._ParseAndComputePenalties(code) + self._CheckPenalties(tree, [ + ('max', None), + ('(', UNBREAKABLE), + ('i', 0), + ('for', 0), + ('i', STRONGLY_CONNECTED), + ('in', STRONGLY_CONNECTED), + ('xrange', STRONGLY_CONNECTED), + ('(', UNBREAKABLE), + ('10', STRONGLY_CONNECTED), + (')', VERY_STRONGLY_CONNECTED), + (')', VERY_STRONGLY_CONNECTED), + ]) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/style_test.py b/yapftests/style_test.py new file mode 100644 index 0000000..7f4a465 --- /dev/null +++ b/yapftests/style_test.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.style.""" + +import shutil +import tempfile +import textwrap +import unittest + +from yapf.yapflib import style + +from yapftests import utils + + +class UtilsTest(unittest.TestCase): + + def testContinuationAlignStyleStringConverter(self): + self.assertEqual(style._ContinuationAlignStyleStringConverter(''), 'SPACE') + self.assertEqual( + style._ContinuationAlignStyleStringConverter('space'), 'SPACE') + self.assertEqual( + style._ContinuationAlignStyleStringConverter('fixed'), 'FIXED') + self.assertEqual( + style._ContinuationAlignStyleStringConverter('valign-right'), + 'VALIGN-RIGHT') + with self.assertRaises(ValueError) as ctx: + style._ContinuationAlignStyleStringConverter('blahblah') + self.assertIn("unknown continuation align style: 'blahblah'", + str(ctx.exception)) + + def testStringListConverter(self): + self.assertEqual(style._StringListConverter('foo, bar'), ['foo', 'bar']) + self.assertEqual(style._StringListConverter('foo,bar'), ['foo', 'bar']) + self.assertEqual(style._StringListConverter(' foo'), ['foo']) + self.assertEqual( + style._StringListConverter('joe ,foo, bar'), ['joe', 'foo', 'bar']) + + def testBoolConverter(self): + self.assertEqual(style._BoolConverter('true'), True) + self.assertEqual(style._BoolConverter('1'), True) + self.assertEqual(style._BoolConverter('false'), False) + self.assertEqual(style._BoolConverter('0'), False) + + +def _LooksLikeChromiumStyle(cfg): + return (cfg['INDENT_WIDTH'] == 2 and + cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + + +def _LooksLikeGoogleStyle(cfg): + return (cfg['INDENT_WIDTH'] == 4 and + cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + + +def _LooksLikePEP8Style(cfg): + return (cfg['INDENT_WIDTH'] == 4 and + not cfg['BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF']) + + +def _LooksLikeFacebookStyle(cfg): + return cfg['INDENT_WIDTH'] == 4 and cfg['DEDENT_CLOSING_BRACKETS'] + + +class PredefinedStylesByNameTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDefault(self): + # default is PEP8 + cfg = style.CreateStyleFromConfig(None) + self.assertTrue(_LooksLikePEP8Style(cfg)) + + def testPEP8ByName(self): + for pep8_name in ('PEP8', 'pep8', 'Pep8'): + cfg = style.CreateStyleFromConfig(pep8_name) + self.assertTrue(_LooksLikePEP8Style(cfg)) + + def testGoogleByName(self): + for google_name in ('google', 'Google', 'GOOGLE'): + cfg = style.CreateStyleFromConfig(google_name) + self.assertTrue(_LooksLikeGoogleStyle(cfg)) + + def testChromiumByName(self): + for chromium_name in ('chromium', 'Chromium', 'CHROMIUM'): + cfg = style.CreateStyleFromConfig(chromium_name) + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + + def testFacebookByName(self): + for fb_name in ('facebook', 'FACEBOOK', 'Facebook'): + cfg = style.CreateStyleFromConfig(fb_name) + self.assertTrue(_LooksLikeFacebookStyle(cfg)) + + +class StyleFromFileTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.test_tmpdir = tempfile.mkdtemp() + style.SetGlobalStyle(style.CreatePEP8Style()) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.test_tmpdir) + + def testDefaultBasedOnStyle(self): + cfg = textwrap.dedent(u'''\ + [style] + continuation_indent_width = 20 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikePEP8Style(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) + + def testDefaultBasedOnPEP8Style(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = pep8 + continuation_indent_width = 40 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikePEP8Style(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 40) + + def testDefaultBasedOnChromiumStyle(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = chromium + continuation_indent_width = 30 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 30) + + def testDefaultBasedOnGoogleStyle(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = google + continuation_indent_width = 20 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikeGoogleStyle(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) + + def testDefaultBasedOnFacebookStyle(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = facebook + continuation_indent_width = 20 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikeFacebookStyle(cfg)) + self.assertEqual(cfg['CONTINUATION_INDENT_WIDTH'], 20) + + def testBoolOptionValue(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = chromium + SPLIT_BEFORE_NAMED_ASSIGNS=False + split_before_logical_operator = true + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['SPLIT_BEFORE_NAMED_ASSIGNS'], False) + self.assertEqual(cfg['SPLIT_BEFORE_LOGICAL_OPERATOR'], True) + + def testStringListOptionValue(self): + cfg = textwrap.dedent(u'''\ + [style] + based_on_style = chromium + I18N_FUNCTION_CALL = N_, V_, T_ + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + cfg = style.CreateStyleFromConfig(filepath) + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['I18N_FUNCTION_CALL'], ['N_', 'V_', 'T_']) + + def testErrorNoStyleFile(self): + with self.assertRaisesRegexp(style.StyleConfigError, + 'is not a valid style or file path'): + style.CreateStyleFromConfig('/8822/xyznosuchfile') + + def testErrorNoStyleSection(self): + cfg = textwrap.dedent(u'''\ + [s] + indent_width=2 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + with self.assertRaisesRegexp(style.StyleConfigError, + 'Unable to find section'): + style.CreateStyleFromConfig(filepath) + + def testErrorUnknownStyleOption(self): + cfg = textwrap.dedent(u'''\ + [style] + indent_width=2 + hummus=2 + ''') + with utils.TempFileContents(self.test_tmpdir, cfg) as filepath: + with self.assertRaisesRegexp(style.StyleConfigError, + 'Unknown style option'): + style.CreateStyleFromConfig(filepath) + + +class StyleFromDict(unittest.TestCase): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDefaultBasedOnStyle(self): + config_dict = { + 'based_on_style': 'pep8', + 'indent_width': 2, + 'blank_line_before_nested_class_or_def': True + } + cfg = style.CreateStyleFromConfig(config_dict) + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['INDENT_WIDTH'], 2) + + def testDefaultBasedOnStyleBadDict(self): + self.assertRaisesRegexp(style.StyleConfigError, 'Unknown style option', + style.CreateStyleFromConfig, + {'based_on_styl': 'pep8'}) + self.assertRaisesRegexp(style.StyleConfigError, 'not a valid', + style.CreateStyleFromConfig, + {'INDENT_WIDTH': 'FOUR'}) + + +class StyleFromCommandLine(unittest.TestCase): + + @classmethod + def setUpClass(cls): + style.SetGlobalStyle(style.CreatePEP8Style()) + + def testDefaultBasedOnStyle(self): + cfg = style.CreateStyleFromConfig( + '{based_on_style: pep8,' + ' indent_width: 2,' + ' blank_line_before_nested_class_or_def: True}') + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['INDENT_WIDTH'], 2) + + def testDefaultBasedOnStyleNotStrict(self): + cfg = style.CreateStyleFromConfig( + '{based_on_style : pep8' + ' ,indent_width=2' + ' blank_line_before_nested_class_or_def:True}') + self.assertTrue(_LooksLikeChromiumStyle(cfg)) + self.assertEqual(cfg['INDENT_WIDTH'], 2) + + def testDefaultBasedOnExplicitlyUnicodeTypeString(self): + cfg = style.CreateStyleFromConfig(u'{}') + self.assertIsInstance(cfg, dict) + + def testDefaultBasedOnDetaultTypeString(self): + cfg = style.CreateStyleFromConfig('{}') + self.assertIsInstance(cfg, dict) + + def testDefaultBasedOnStyleBadString(self): + self.assertRaisesRegexp(style.StyleConfigError, 'Unknown style option', + style.CreateStyleFromConfig, + '{based_on_styl: pep8}') + self.assertRaisesRegexp(style.StyleConfigError, 'not a valid', + style.CreateStyleFromConfig, '{INDENT_WIDTH: FOUR}') + self.assertRaisesRegexp(style.StyleConfigError, 'Invalid style dict', + style.CreateStyleFromConfig, + '{based_on_style: pep8') + + +class StyleHelp(unittest.TestCase): + + def testHelpKeys(self): + settings = sorted(style.Help()) + expected = sorted(style._style) + self.assertListEqual(settings, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/subtype_assigner_test.py b/yapftests/subtype_assigner_test.py new file mode 100644 index 0000000..8daead9 --- /dev/null +++ b/yapftests/subtype_assigner_test.py @@ -0,0 +1,200 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.subtype_assigner.""" + +import textwrap +import unittest + +from yapf.yapflib import format_token +from yapf.yapflib import pytree_utils + +from yapftests import yapf_test_helper + + +class SubtypeAssignerTest(yapf_test_helper.YAPFTest): + + def _CheckFormatTokenSubtypes(self, uwlines, list_of_expected): + """Check that the tokens in the UnwrappedLines have the expected subtypes. + + Args: + uwlines: list of UnwrappedLine. + list_of_expected: list of (name, subtype) pairs. Non-semantic tokens are + filtered out from the expected values. + """ + actual = [] + for uwl in uwlines: + filtered_values = [(ft.value, ft.subtypes) + for ft in uwl.tokens + if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] + if filtered_values: + actual.append(filtered_values) + + self.assertEqual(list_of_expected, actual) + + def testFuncDefDefaultAssign(self): + code = textwrap.dedent(r""" + def foo(a=37, *b, **c): + return -x[:42] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('def', [format_token.Subtype.NONE]), + ('foo', {format_token.Subtype.FUNC_DEF}), + ('(', [format_token.Subtype.NONE]), + ('a', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('=', {format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('37', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + (',', {format_token.Subtype.NONE}), + ('*', {format_token.Subtype.VARARGS_STAR, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('b', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + (',', {format_token.Subtype.NONE}), + ('**', {format_token.Subtype.KWARGS_STAR_STAR, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('c', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + (')', [format_token.Subtype.NONE]), + (':', [format_token.Subtype.NONE])], + [('return', [format_token.Subtype.NONE]), + ('-', {format_token.Subtype.UNARY_OPERATOR}), + ('x', [format_token.Subtype.NONE]), + ('[', {format_token.Subtype.SUBSCRIPT_BRACKET}), + (':', {format_token.Subtype.SUBSCRIPT_COLON}), + ('42', [format_token.Subtype.NONE]), + (']', {format_token.Subtype.SUBSCRIPT_BRACKET})], + ]) # yapf: disable + + def testFuncCallWithDefaultAssign(self): + code = textwrap.dedent(r""" + foo(x, a='hello world') + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('foo', [format_token.Subtype.NONE]), + ('(', [format_token.Subtype.NONE]), + ('x', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + (',', {format_token.Subtype.NONE}), + ('a', {format_token.Subtype.NONE, + format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST}), + ('=', {format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN}), + ("'hello world'", {format_token.Subtype.NONE}), + (')', [format_token.Subtype.NONE])], + ]) # yapf: disable + + def testSetComprehension(self): + code = textwrap.dedent("""\ + def foo(strs): + return {s.lower() for s in strs} + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('def', [format_token.Subtype.NONE]), + ('foo', {format_token.Subtype.FUNC_DEF}), + ('(', [format_token.Subtype.NONE]), + ('strs', [format_token.Subtype.NONE]), + (')', [format_token.Subtype.NONE]), + (':', [format_token.Subtype.NONE])], + [('return', [format_token.Subtype.NONE]), + ('{', [format_token.Subtype.NONE]), + ('s', {format_token.Subtype.COMP_EXPR}), + ('.', {format_token.Subtype.COMP_EXPR}), + ('lower', {format_token.Subtype.COMP_EXPR}), + ('(', {format_token.Subtype.COMP_EXPR}), + (')', {format_token.Subtype.COMP_EXPR}), + ('for', {format_token.Subtype.DICT_SET_GENERATOR, + format_token.Subtype.COMP_FOR}), + ('s', {format_token.Subtype.COMP_FOR}), + ('in', {format_token.Subtype.COMP_FOR}), + ('strs', {format_token.Subtype.COMP_FOR}), + ('}', [format_token.Subtype.NONE])] + ]) # yapf: disable + + def testUnaryNotOperator(self): + code = textwrap.dedent("""\ + not a + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('not', {format_token.Subtype.UNARY_OPERATOR}), + ('a', [format_token.Subtype.NONE])] + ]) # yapf: disable + + def testBitwiseOperators(self): + code = textwrap.dedent("""\ + x = ((a | (b ^ 3) & c) << 3) >> 1 + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('x', [format_token.Subtype.NONE]), + ('=', {format_token.Subtype.ASSIGN_OPERATOR}), + ('(', [format_token.Subtype.NONE]), + ('(', [format_token.Subtype.NONE]), + ('a', [format_token.Subtype.NONE]), + ('|', {format_token.Subtype.BINARY_OPERATOR}), + ('(', [format_token.Subtype.NONE]), + ('b', [format_token.Subtype.NONE]), + ('^', {format_token.Subtype.BINARY_OPERATOR}), + ('3', [format_token.Subtype.NONE]), + (')', [format_token.Subtype.NONE]), + ('&', {format_token.Subtype.BINARY_OPERATOR}), + ('c', [format_token.Subtype.NONE]), + (')', [format_token.Subtype.NONE]), + ('<<', {format_token.Subtype.BINARY_OPERATOR}), + ('3', [format_token.Subtype.NONE]), + (')', [format_token.Subtype.NONE]), + ('>>', {format_token.Subtype.BINARY_OPERATOR}), + ('1', [format_token.Subtype.NONE]),], + ]) # yapf: disable + + def testSubscriptColon(self): + code = textwrap.dedent("""\ + x[0:42:1] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [ + ('x', [format_token.Subtype.NONE]), + ('[', {format_token.Subtype.SUBSCRIPT_BRACKET}), + ('0', [format_token.Subtype.NONE]), + (':', {format_token.Subtype.SUBSCRIPT_COLON}), + ('42', [format_token.Subtype.NONE]), + (':', {format_token.Subtype.SUBSCRIPT_COLON}), + ('1', [format_token.Subtype.NONE]), + (']', {format_token.Subtype.SUBSCRIPT_BRACKET}), + ], + ]) + + def testFunctionCallWithStarExpression(self): + code = textwrap.dedent("""\ + [a, *b] + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + self._CheckFormatTokenSubtypes(uwlines, [ + [('[', [format_token.Subtype.NONE]), + ('a', [format_token.Subtype.NONE]), + (',', [format_token.Subtype.NONE]), + ('*', {format_token.Subtype.UNARY_OPERATOR, + format_token.Subtype.VARARGS_STAR}), + ('b', [format_token.Subtype.NONE]), + (']', [format_token.Subtype.NONE]),], + ]) # yapf: disable + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/unwrapped_line_test.py b/yapftests/unwrapped_line_test.py new file mode 100644 index 0000000..90be1a1 --- /dev/null +++ b/yapftests/unwrapped_line_test.py @@ -0,0 +1,96 @@ +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.unwrapped_line.""" + +import textwrap +import unittest + +from lib2to3 import pytree +from lib2to3.pgen2 import token + +from yapf.yapflib import format_token +from yapf.yapflib import split_penalty +from yapf.yapflib import unwrapped_line + +from yapftests import yapf_test_helper + + +class UnwrappedLineBasicTest(unittest.TestCase): + + def testConstruction(self): + toks = _MakeFormatTokenList([(token.DOT, '.'), (token.VBAR, '|')]) + uwl = unwrapped_line.UnwrappedLine(20, toks) + self.assertEqual(20, uwl.depth) + self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in uwl.tokens]) + + def testFirstLast(self): + toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), + (token.VBAR, '|')]) + uwl = unwrapped_line.UnwrappedLine(20, toks) + self.assertEqual(20, uwl.depth) + self.assertEqual('DOT', uwl.first.name) + self.assertEqual('VBAR', uwl.last.name) + + def testAsCode(self): + toks = _MakeFormatTokenList([(token.DOT, '.'), (token.LPAR, '('), + (token.VBAR, '|')]) + uwl = unwrapped_line.UnwrappedLine(2, toks) + self.assertEqual(' . ( |', uwl.AsCode()) + + def testAppendToken(self): + uwl = unwrapped_line.UnwrappedLine(0) + uwl.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '(')) + uwl.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')')) + self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens]) + + def testAppendNode(self): + uwl = unwrapped_line.UnwrappedLine(0) + uwl.AppendNode(pytree.Leaf(token.LPAR, '(')) + uwl.AppendNode(pytree.Leaf(token.RPAR, ')')) + self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in uwl.tokens]) + + +class UnwrappedLineFormattingInformationTest(yapf_test_helper.YAPFTest): + + def testFuncDef(self): + code = textwrap.dedent(r""" + def f(a, b): + pass + """) + uwlines = yapf_test_helper.ParseAndUnwrap(code) + + f = uwlines[0].tokens[1] + self.assertFalse(f.can_break_before) + self.assertFalse(f.must_break_before) + self.assertEqual(f.split_penalty, split_penalty.UNBREAKABLE) + + lparen = uwlines[0].tokens[2] + self.assertFalse(lparen.can_break_before) + self.assertFalse(lparen.must_break_before) + self.assertEqual(lparen.split_penalty, split_penalty.UNBREAKABLE) + + +def _MakeFormatTokenLeaf(token_type, token_value): + return format_token.FormatToken(pytree.Leaf(token_type, token_value)) + + +def _MakeFormatTokenList(token_type_values): + return [ + _MakeFormatTokenLeaf(token_type, token_value) + for token_type, token_value in token_type_values + ] + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/utils.py b/yapftests/utils.py new file mode 100644 index 0000000..268b8c4 --- /dev/null +++ b/yapftests/utils.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for tests.""" + +import contextlib +import io +import os +import sys +import tempfile + + +@contextlib.contextmanager +def stdout_redirector(stream): # pylint: disable=invalid-name + old_stdout = sys.stdout + sys.stdout = stream + try: + yield + finally: + sys.stdout = old_stdout + + +# NamedTemporaryFile is useless because on Windows the temporary file would be +# created with O_TEMPORARY, which would not allow the file to be opened a +# second time, even by the same process, unless the same flag is used. +# Thus we provide a simplified version ourselves. +# +# Note: returns a tuple of (io.file_obj, file_path), instead of a file_obj with +# a .name attribute +# +# Note: `buffering` is set to -1 despite documentation of NamedTemporaryFile +# says None. This is probably a problem with the python documentation. +@contextlib.contextmanager +def NamedTempFile(mode='w+b', + buffering=-1, + encoding=None, + errors=None, + newline=None, + suffix=None, + prefix=None, + dirname=None, + text=False): + """Context manager creating a new temporary file in text mode.""" + if sys.version_info < (3, 5): # covers also python 2 + if suffix is None: + suffix = '' + if prefix is None: + prefix = 'tmp' + (fd, fname) = tempfile.mkstemp( + suffix=suffix, prefix=prefix, dir=dirname, text=text) + f = io.open( + fd, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline) + yield f, fname + f.close() + os.remove(fname) + + +@contextlib.contextmanager +def TempFileContents(dirname, + contents, + encoding='utf-8', + newline='', + suffix=None): + # Note: NamedTempFile properly handles unicode encoding when using mode='w' + with NamedTempFile( + dirname=dirname, + mode='w', + encoding=encoding, + newline=newline, + suffix=suffix) as (f, fname): + f.write(contents) + f.flush() + yield fname diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py new file mode 100644 index 0000000..6df28b6 --- /dev/null +++ b/yapftests/yapf_test.py @@ -0,0 +1,1393 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for yapf.yapf.""" + +import io +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest + +from yapf.yapflib import py3compat +from yapf.yapflib import style +from yapf.yapflib import yapf_api + +from yapftests import utils + +ROOT_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) + +# Verification is turned off by default, but want to enable it for testing. +YAPF_BINARY = [sys.executable, '-m', 'yapf', '--verify', '--no-local-style'] + + +class FormatCodeTest(unittest.TestCase): + + def _Check(self, unformatted_code, expected_formatted_code): + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config='chromium') + self.assertEqual(expected_formatted_code, formatted_code) + + def testSimple(self): + unformatted_code = textwrap.dedent("""\ + print('foo') + """) + self._Check(unformatted_code, unformatted_code) + + def testNoEndingNewline(self): + unformatted_code = textwrap.dedent("""\ + if True: + pass""") + expected_formatted_code = textwrap.dedent("""\ + if True: + pass + """) + self._Check(unformatted_code, expected_formatted_code) + + +class FormatFileTest(unittest.TestCase): + + def setUp(self): + self.test_tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.test_tmpdir) + + def assertCodeEqual(self, expected_code, code): + if code != expected_code: + msg = 'Code format mismatch:\n' + msg += 'Expected:\n >' + msg += '\n > '.join(expected_code.splitlines()) + msg += '\nActual:\n >' + msg += '\n > '.join(code.splitlines()) + # TODO(sbc): maybe using difflib here to produce easy to read deltas? + self.fail(msg) + + def testFormatFile(self): + unformatted_code = textwrap.dedent(u"""\ + if True: + pass + """) + expected_formatted_code_pep8 = textwrap.dedent(u"""\ + if True: + pass + """) + expected_formatted_code_chromium = textwrap.dedent(u"""\ + if True: + pass + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code_pep8, formatted_code) + + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='chromium') + self.assertCodeEqual(expected_formatted_code_chromium, formatted_code) + + def testDisableLinesPattern(self): + unformatted_code = textwrap.dedent(u"""\ + if a: b + + # yapf: disable + if f: g + + if h: i + """) + expected_formatted_code = textwrap.dedent(u"""\ + if a: b + + # yapf: disable + if f: g + + if h: i + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testDisableAndReenableLinesPattern(self): + unformatted_code = textwrap.dedent(u"""\ + if a: b + + # yapf: disable + if f: g + # yapf: enable + + if h: i + """) + expected_formatted_code = textwrap.dedent(u"""\ + if a: b + + # yapf: disable + if f: g + # yapf: enable + + if h: i + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testDisablePartOfMultilineComment(self): + unformatted_code = textwrap.dedent(u"""\ + if a: b + + # This is a multiline comment that disables YAPF. + # yapf: disable + if f: g + # yapf: enable + # This is a multiline comment that enables YAPF. + + if h: i + """) + + expected_formatted_code = textwrap.dedent(u"""\ + if a: b + + # This is a multiline comment that disables YAPF. + # yapf: disable + if f: g + # yapf: enable + # This is a multiline comment that enables YAPF. + + if h: i + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + code = textwrap.dedent(u"""\ + def foo_function(): + # some comment + # yapf: disable + + foo( + bar, + baz + ) + + # yapf: enable + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testFormatFileLinesSelection(self): + unformatted_code = textwrap.dedent(u"""\ + if a: b + + if f: g + + if h: i + """) + expected_formatted_code_lines1and2 = textwrap.dedent(u"""\ + if a: b + + if f: g + + if h: i + """) + expected_formatted_code_lines3 = textwrap.dedent(u"""\ + if a: b + + if f: g + + if h: i + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='pep8', lines=[(1, 2)]) + self.assertCodeEqual(expected_formatted_code_lines1and2, formatted_code) + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='pep8', lines=[(3, 3)]) + self.assertCodeEqual(expected_formatted_code_lines3, formatted_code) + + def testFormatFileDiff(self): + unformatted_code = textwrap.dedent(u"""\ + if True: + pass + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + diff, _, _ = yapf_api.FormatFile(filepath, print_diff=True) + self.assertTrue(u'+ pass' in diff) + + def testFormatFileInPlace(self): + unformatted_code = u'True==False\n' + formatted_code = u'True == False\n' + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + result, _, _ = yapf_api.FormatFile(filepath, in_place=True) + self.assertEqual(result, None) + with open(filepath) as fd: + if sys.version_info[0] <= 2: + self.assertCodeEqual(formatted_code, fd.read().decode('ascii')) + else: + self.assertCodeEqual(formatted_code, fd.read()) + + self.assertRaises( + ValueError, + yapf_api.FormatFile, + filepath, + in_place=True, + print_diff=True) + + def testNoFile(self): + stream = py3compat.StringIO() + handler = logging.StreamHandler(stream) + logger = logging.getLogger('mylogger') + logger.addHandler(handler) + self.assertRaises( + IOError, yapf_api.FormatFile, 'not_a_file.py', logger=logger.error) + self.assertEqual(stream.getvalue(), + "[Errno 2] No such file or directory: 'not_a_file.py'\n") + + def testCommentsUnformatted(self): + code = textwrap.dedent(u"""\ + foo = [# A list of things + # bork + 'one', + # quark + 'two'] # yapf: disable + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testDisabledHorizontalFormattingOnNewLine(self): + code = textwrap.dedent(u"""\ + # yapf: disable + a = [ + 1] + # yapf: enable + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testSplittingSemicolonStatements(self): + unformatted_code = textwrap.dedent(u"""\ + def f(): + x = y + 42 ; z = n * 42 + if True: a += 1 ; b += 1; c += 1 + """) + expected_formatted_code = textwrap.dedent(u"""\ + def f(): + x = y + 42 + z = n * 42 + if True: + a += 1 + b += 1 + c += 1 + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testSemicolonStatementsDisabled(self): + unformatted_code = textwrap.dedent(u"""\ + def f(): + x = y + 42 ; z = n * 42 # yapf: disable + if True: a += 1 ; b += 1; c += 1 + """) + expected_formatted_code = textwrap.dedent(u"""\ + def f(): + x = y + 42 ; z = n * 42 # yapf: disable + if True: + a += 1 + b += 1 + c += 1 + """) + with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(expected_formatted_code, formatted_code) + + def testDisabledSemiColonSeparatedStatements(self): + code = textwrap.dedent(u"""\ + # yapf: disable + if True: a ; b + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8') + self.assertCodeEqual(code, formatted_code) + + def testDisabledMultilineStringInDictionary(self): + code = textwrap.dedent(u"""\ + # yapf: disable + + A = [ + { + "aaaaaaaaaaaaaaaaaaa": ''' + bbbbbbbbbbb: "ccccccccccc" + dddddddddddddd: 1 + eeeeeeee: 0 + ffffffffff: "ggggggg" + ''', + }, + ] + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='chromium') + self.assertCodeEqual(code, formatted_code) + + def testDisabledWithPrecedingText(self): + code = textwrap.dedent(u"""\ + # TODO(fix formatting): yapf: disable + + A = [ + { + "aaaaaaaaaaaaaaaaaaa": ''' + bbbbbbbbbbb: "ccccccccccc" + dddddddddddddd: 1 + eeeeeeee: 0 + ffffffffff: "ggggggg" + ''', + }, + ] + """) + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='chromium') + self.assertCodeEqual(code, formatted_code) + + def testCRLFLineEnding(self): + code = u'class _():\r\n pass\r\n' + with utils.TempFileContents(self.test_tmpdir, code) as filepath: + formatted_code, _, _ = yapf_api.FormatFile( + filepath, style_config='chromium') + self.assertCodeEqual(code, formatted_code) + + +class CommandLineTest(unittest.TestCase): + """Test how calling yapf from the command line acts.""" + + @classmethod + def setUpClass(cls): + cls.test_tmpdir = tempfile.mkdtemp() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.test_tmpdir) + + def assertYapfReformats(self, + unformatted, + expected, + extra_options=None, + env=None): + """Check that yapf reformats the given code as expected. + + Invokes yapf in a subprocess, piping the unformatted code into its stdin. + Checks that the formatted output is as expected. + + Arguments: + unformatted: unformatted code - input to yapf + expected: expected formatted code at the output of yapf + extra_options: iterable of extra command-line options to pass to yapf + env: dict of environment variables. + """ + cmdline = YAPF_BINARY + (extra_options or []) + p = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + reformatted_code, stderrdata = p.communicate(unformatted.encode('utf-8')) + self.assertEqual(stderrdata, b'') + self.assertMultiLineEqual(reformatted_code.decode('utf-8'), expected) + + def testUnicodeEncodingPipedToFile(self): + unformatted_code = textwrap.dedent(u"""\ + def foo(): + print('⇒') + """) + with utils.NamedTempFile( + dirname=self.test_tmpdir, suffix='.py') as (out, _): + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + subprocess.check_call(YAPF_BINARY + ['--diff', filepath], stdout=out) + + def testInPlaceReformatting(self): + unformatted_code = textwrap.dedent(u"""\ + def foo(): + x = 37 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + x = 37 + """) + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) + p.wait() + with io.open(filepath, mode='r', newline='') as fd: + reformatted_code = fd.read() + self.assertEqual(reformatted_code, expected_formatted_code) + + def testInPlaceReformattingBlank(self): + unformatted_code = u'\n\n' + expected_formatted_code = u'\n' + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) + p.wait() + with io.open(filepath, mode='r', encoding='utf-8', newline='') as fd: + reformatted_code = fd.read() + self.assertEqual(reformatted_code, expected_formatted_code) + + def testInPlaceReformattingEmpty(self): + unformatted_code = u'' + expected_formatted_code = u'' + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + p = subprocess.Popen(YAPF_BINARY + ['--in-place', filepath]) + p.wait() + with io.open(filepath, mode='r', encoding='utf-8', newline='') as fd: + reformatted_code = fd.read() + self.assertEqual(reformatted_code, expected_formatted_code) + + def testReadFromStdin(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + x = 37 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + x = 37 + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testReadFromStdinWithEscapedStrings(self): + unformatted_code = textwrap.dedent("""\ + s = "foo\\nbar" + """) + expected_formatted_code = textwrap.dedent("""\ + s = "foo\\nbar" + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testSetChromiumStyle(self): + unformatted_code = textwrap.dedent("""\ + def foo(): # trail + x = 37 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): # trail + x = 37 + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style=chromium']) + + def testSetCustomStyleBasedOnChromium(self): + unformatted_code = textwrap.dedent("""\ + def foo(): # trail + x = 37 + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): # trail + x = 37 + """) + style_file = textwrap.dedent(u'''\ + [style] + based_on_style = chromium + SPACES_BEFORE_COMMENT = 4 + ''') + with utils.TempFileContents(self.test_tmpdir, style_file) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testReadSingleLineCodeFromStdin(self): + unformatted_code = textwrap.dedent("""\ + if True: pass + """) + expected_formatted_code = textwrap.dedent("""\ + if True: pass + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testEncodingVerification(self): + unformatted_code = textwrap.dedent(u"""\ + '''The module docstring.''' + # -*- coding: utf-8 -*- + def f(): + x = 37 + """) + + with utils.NamedTempFile( + suffix='.py', dirname=self.test_tmpdir) as (out, _): + with utils.TempFileContents( + self.test_tmpdir, unformatted_code, suffix='.py') as filepath: + try: + subprocess.check_call(YAPF_BINARY + ['--diff', filepath], stdout=out) + except subprocess.CalledProcessError as e: + self.assertEqual(e.returncode, 1) # Indicates the text changed. + + def testReformattingSpecificLines(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + """) + # TODO(ambv): the `expected_formatted_code` here is not PEP8 compliant, + # raising "E129 visually indented line with same indent as next logical + # line" with flake8. + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-2']) + + def testOmitFormattingLinesBeforeDisabledFunctionComment(self): + unformatted_code = textwrap.dedent("""\ + import sys + + # Comment + def some_func(x): + x = ["badly" , "formatted","line" ] + """) + expected_formatted_code = textwrap.dedent("""\ + import sys + + # Comment + def some_func(x): + x = ["badly", "formatted", "line"] + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '5-5']) + + def testReformattingSkippingLines(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + # yapf: disable + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + # yapf: enable + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + # yapf: disable + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + # yapf: enable + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testReformattingSkippingToEndOfFile(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + # yapf: disable + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def f(): + def e(): + while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == + 'bbbbbbb'): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + # yapf: disable + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def f(): + def e(): + while (xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxxxxxxxxxxx(yyyyyyyyyyyyy[zzzzz].aaaaaaaa[0]) == + 'bbbbbbb'): + pass + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testReformattingSkippingSingleLine(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable + pass + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testDisableWholeDataStructure(self): + unformatted_code = textwrap.dedent("""\ + A = set([ + 'hello', + 'world', + ]) # yapf: disable + """) + expected_formatted_code = textwrap.dedent("""\ + A = set([ + 'hello', + 'world', + ]) # yapf: disable + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testDisableButAdjustIndentations(self): + unformatted_code = textwrap.dedent("""\ + class SplitPenaltyTest(unittest.TestCase): + def testUnbreakable(self): + self._CheckPenalties(tree, [ + ]) # yapf: disable + """) + expected_formatted_code = textwrap.dedent("""\ + class SplitPenaltyTest(unittest.TestCase): + def testUnbreakable(self): + self._CheckPenalties(tree, [ + ]) # yapf: disable + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testRetainingHorizontalWhitespace(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy (zzzzzzzzzzzzz [0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + + def g(): + if (xxxxxxxxxxxx.yyyyyyyy (zzzzzzzzzzzzz [0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): # yapf: disable + pass + """) + self.assertYapfReformats(unformatted_code, expected_formatted_code) + + def testRetainingVerticalWhitespace(self): + unformatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def g(): + + + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def h(): + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and + xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + pass + + def g(): + + + if (xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0]) == 'aaaaaaaaaaa' and xxxxxxxxxxxx.yyyyyyyy(zzzzzzzzzzzzz[0].mmmmmmmm[0]) == 'bbbbbbb'): + + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-2']) + + unformatted_code = textwrap.dedent("""\ + + + if a: b + + + if c: + to_much + indent + + same + + + + #comment + + # trailing whitespace + """) + expected_formatted_code = textwrap.dedent("""\ + if a: b + + + if c: + to_much + indent + + same + + + + #comment + + # trailing whitespace + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '3-3', '--lines', '13-13']) + + unformatted_code = textwrap.dedent("""\ + ''' + docstring + + ''' + + import blah + """) + + self.assertYapfReformats( + unformatted_code, unformatted_code, extra_options=['--lines', '2-2']) + + def testRetainingSemicolonsWhenSpecifyingLines(self): + unformatted_code = textwrap.dedent("""\ + a = line_to_format + def f(): + x = y + 42; z = n * 42 + if True: a += 1 ; b += 1 ; c += 1 + """) + expected_formatted_code = textwrap.dedent("""\ + a = line_to_format + + + def f(): + x = y + 42; z = n * 42 + if True: a += 1 ; b += 1 ; c += 1 + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1']) + + def testDisabledMultilineStrings(self): + unformatted_code = textwrap.dedent('''\ + foo=42 + def f(): + email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br> + <b>Czar: </b>"""+despot["Nicholas"]+"""<br> + <b>Minion: </b>"""+serf["Dmitri"]+"""<br> + <b>Residence: </b>"""+palace["Winter"]+"""<br> + </body> + </html>""" + ''') + expected_formatted_code = textwrap.dedent('''\ + foo = 42 + + + def f(): + email_text += """<html>This is a really long docstring that goes over the column limit and is multi-line.<br><br> + <b>Czar: </b>"""+despot["Nicholas"]+"""<br> + <b>Minion: </b>"""+serf["Dmitri"]+"""<br> + <b>Residence: </b>"""+palace["Winter"]+"""<br> + </body> + </html>""" + ''') + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1']) + + def testDisableWhenSpecifyingLines(self): + unformatted_code = textwrap.dedent("""\ + # yapf: disable + A = set([ + 'hello', + 'world', + ]) + # yapf: enable + B = set([ + 'hello', + 'world', + ]) # yapf: disable + """) + expected_formatted_code = textwrap.dedent("""\ + # yapf: disable + A = set([ + 'hello', + 'world', + ]) + # yapf: enable + B = set([ + 'hello', + 'world', + ]) # yapf: disable + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-10']) + + def testDisableFormattingInDataLiteral(self): + unformatted_code = textwrap.dedent("""\ + def horrible(): + oh_god() + why_would_you() + [ + 'do', + + 'that', + ] + + def still_horrible(): + oh_god() + why_would_you() + [ + 'do', + + 'that' + ] + """) + expected_formatted_code = textwrap.dedent("""\ + def horrible(): + oh_god() + why_would_you() + [ + 'do', + + 'that', + ] + + def still_horrible(): + oh_god() + why_would_you() + ['do', 'that'] + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '14-15']) + + def testRetainVerticalFormattingBetweenDisabledAndEnabledLines(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) + self.assertEqual( + ('ddddddddddddddddddddddddd', + 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % + c.ffffffffffff), + gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) + iiiii = jjjjjjjjjjjjjj.iiiii + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) + self.assertEqual(('ddddddddddddddddddddddddd', + 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % c.ffffffffffff), + gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) + iiiii = jjjjjjjjjjjjjj.iiiii + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '4-7']) + + def testFormatLinesSpecifiedInMiddleOfExpression(self): + unformatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) + self.assertEqual( + ('ddddddddddddddddddddddddd', + 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % + c.ffffffffffff), + gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) + iiiii = jjjjjjjjjjjjjj.iiiii + """) + expected_formatted_code = textwrap.dedent("""\ + class A(object): + def aaaaaaaaaaaaa(self): + c = bbbbbbbbb.ccccccccc('challenge', 0, 1, 10) + self.assertEqual(('ddddddddddddddddddddddddd', + 'eeeeeeeeeeeeeeeeeeeeeeeee.%s' % c.ffffffffffff), + gggggggggggg.hhhhhhhhh(c, c.ffffffffffff)) + iiiii = jjjjjjjjjjjjjj.iiiii + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '5-6']) + + def testCommentFollowingMultilineString(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + '''First line. + Second line. + ''' # comment + x = '''hello world''' # second comment + return 42 # another comment + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + '''First line. + Second line. + ''' # comment + x = '''hello world''' # second comment + return 42 # another comment + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1']) + + def testDedentClosingBracket(self): + # no line-break on the first argument, not dedenting closing brackets + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name(first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def overly_long_function_name(first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style=pep8']) + + # TODO(ambv): currently the following produces the closing bracket on a new + # line but indented to the opening bracket which is the worst of both + # worlds. Expected behaviour would be to format as --style=pep8 does in + # this case. + # self.assertYapfReformats(unformatted_code, expected_formatted_code, + # extra_options=['--style=facebook']) + + # line-break before the first argument, dedenting closing brackets if set + unformatted_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, + second_argument_makes_the_line_too_long): + pass + """) + # expected_formatted_pep8_code = textwrap.dedent("""\ + # def overly_long_function_name( + # first_argument_on_the_same_line, + # second_argument_makes_the_line_too_long): + # pass + # """) + expected_formatted_fb_code = textwrap.dedent("""\ + def overly_long_function_name( + first_argument_on_the_same_line, second_argument_makes_the_line_too_long + ): + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_fb_code, + extra_options=['--style=facebook']) + # TODO(ambv): currently the following produces code that is not PEP8 + # compliant, raising "E125 continuation line with same indent as next + # logical line" with flake8. Expected behaviour for PEP8 would be to use + # double-indentation here. + # self.assertYapfReformats(unformatted_code, expected_formatted_pep8_code, + # extra_options=['--style=pep8']) + + def testCoalesceBrackets(self): + unformatted_code = textwrap.dedent("""\ + some_long_function_name_foo( + { + 'first_argument_of_the_thing': id, + 'second_argument_of_the_thing': "some thing" + } + )""") + expected_formatted_code = textwrap.dedent("""\ + some_long_function_name_foo({ + 'first_argument_of_the_thing': id, + 'second_argument_of_the_thing': "some thing" + }) + """) + with utils.NamedTempFile(dirname=self.test_tmpdir, mode='w') as (f, name): + f.write( + textwrap.dedent(u'''\ + [style] + column_limit=82 + coalesce_brackets = True + ''')) + f.flush() + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(name)]) + + def testPseudoParenSpaces(self): + unformatted_code = textwrap.dedent("""\ + def foo(): + def bar(): + return {msg_id: author for author, msg_id in reader} + """) + expected_formatted_code = textwrap.dedent("""\ + def foo(): + + def bar(): + return {msg_id: author for author, msg_id in reader} + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1', '--style', 'chromium']) + + def testMultilineCommentFormattingDisabled(self): + unformatted_code = textwrap.dedent("""\ + # This is a comment + FOO = { + aaaaaaaa.ZZZ: [ + bbbbbbbbbb.Pop(), + # Multiline comment. + # Line two. + bbbbbbbbbb.Pop(), + ], + 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': + ('yyyyy', zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz), + '#': lambda x: x # do nothing + } + """) + expected_formatted_code = textwrap.dedent("""\ + # This is a comment + FOO = { + aaaaaaaa.ZZZ: [ + bbbbbbbbbb.Pop(), + # Multiline comment. + # Line two. + bbbbbbbbbb.Pop(), + ], + 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': + ('yyyyy', zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz), + '#': lambda x: x # do nothing + } + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1', '--style', 'chromium']) + + def testTrailingCommentsWithDisabledFormatting(self): + unformatted_code = textwrap.dedent("""\ + import os + + SCOPES = [ + 'hello world' # This is a comment. + ] + """) + expected_formatted_code = textwrap.dedent("""\ + import os + + SCOPES = [ + 'hello world' # This is a comment. + ] + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-1', '--style', 'chromium']) + + def testUseTabs(self): + unformatted_code = """\ +def foo_function(): + if True: + pass +""" + expected_formatted_code = """\ +def foo_function(): + if True: + pass +""" + style_contents = u"""\ +[style] +based_on_style = chromium +USE_TABS = true +INDENT_WIDTH=1 +""" + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsWith(self): + unformatted_code = """\ +def f(): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def f(): + return [ + 'hello', + 'world', + ] +""" + style_contents = u"""\ +[style] +based_on_style = chromium +USE_TABS = true +INDENT_WIDTH=1 +""" + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsContinuationAlignStyleFixed(self): + unformatted_code = """\ +def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def foo_function(arg1, arg2, + arg3): + return [ + 'hello', + 'world', + ] +""" + style_contents = u"""\ +[style] +based_on_style = chromium +USE_TABS = true +COLUMN_LIMIT=32 +INDENT_WIDTH=4 +CONTINUATION_INDENT_WIDTH=8 +CONTINUATION_ALIGN_STYLE = fixed +""" + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testUseTabsContinuationAlignStyleVAlignRight(self): + unformatted_code = """\ +def foo_function(arg1, arg2, arg3): + return ['hello', 'world',] +""" + expected_formatted_code = """\ +def foo_function(arg1, arg2, + arg3): + return [ + 'hello', + 'world', + ] +""" + style_contents = u"""\ +[style] +based_on_style = chromium +USE_TABS = true +COLUMN_LIMIT=32 +INDENT_WIDTH=4 +CONTINUATION_INDENT_WIDTH=8 +CONTINUATION_ALIGN_STYLE = valign-right +""" + with utils.TempFileContents(self.test_tmpdir, style_contents) as stylepath: + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testStyleOutputRoundTrip(self): + unformatted_code = textwrap.dedent("""\ + def foo_function(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + def foo_function(): + pass + """) + + with utils.NamedTempFile(dirname=self.test_tmpdir) as (stylefile, + stylepath): + p = subprocess.Popen( + YAPF_BINARY + ['--style-help'], + stdout=stylefile, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE) + _, stderrdata = p.communicate() + self.assertEqual(stderrdata, b'') + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style={0}'.format(stylepath)]) + + def testSpacingBeforeComments(self): + unformatted_code = textwrap.dedent("""\ + A = 42 + + + # A comment + def x(): + pass + def _(): + pass + """) + expected_formatted_code = textwrap.dedent("""\ + A = 42 + + + # A comment + def x(): + pass + def _(): + pass + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--lines', '1-2']) + + def testSpacingBeforeCommentsInDicts(self): + unformatted_code = textwrap.dedent("""\ + A=42 + + X = { + # 'Valid' statuses. + PASSED: # Passed + 'PASSED', + FAILED: # Failed + 'FAILED', + TIMED_OUT: # Timed out. + 'FAILED', + BORKED: # Broken. + 'BROKEN' + } + """) + expected_formatted_code = textwrap.dedent("""\ + A = 42 + + X = { + # 'Valid' statuses. + PASSED: # Passed + 'PASSED', + FAILED: # Failed + 'FAILED', + TIMED_OUT: # Timed out. + 'FAILED', + BORKED: # Broken. + 'BROKEN' + } + """) + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + extra_options=['--style', 'chromium', '--lines', '1-1']) + + @unittest.skipUnless(py3compat.PY36, 'Requires Python 3.6') + def testCP936Encoding(self): + unformatted_code = 'print("中文")\n' + expected_formatted_code = 'print("中文")\n' + self.assertYapfReformats( + unformatted_code, + expected_formatted_code, + env={'PYTHONIOENCODING': 'cp936'}) + + +class BadInputTest(unittest.TestCase): + """Test yapf's behaviour when passed bad input.""" + + def testBadSyntax(self): + code = ' a = 1\n' + self.assertRaises(SyntaxError, yapf_api.FormatCode, code) + + +class DiffIndentTest(unittest.TestCase): + + @staticmethod + def _OwnStyle(): + my_style = style.CreatePEP8Style() + my_style['INDENT_WIDTH'] = 3 + my_style['CONTINUATION_INDENT_WIDTH'] = 3 + return my_style + + def _Check(self, unformatted_code, expected_formatted_code): + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config=style.SetGlobalStyle(self._OwnStyle())) + self.assertEqual(expected_formatted_code, formatted_code) + + def testSimple(self): + unformatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + expected_formatted_code = textwrap.dedent("""\ + for i in range(5): + print('bar') + """) + self._Check(unformatted_code, expected_formatted_code) + + +if __name__ == '__main__': + unittest.main() diff --git a/yapftests/yapf_test_helper.py b/yapftests/yapf_test_helper.py new file mode 100644 index 0000000..1f21b36 --- /dev/null +++ b/yapftests/yapf_test_helper.py @@ -0,0 +1,89 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Support module for tests for yapf.""" + +import difflib +import sys +import unittest + +from yapf.yapflib import blank_line_calculator +from yapf.yapflib import comment_splicer +from yapf.yapflib import continuation_splicer +from yapf.yapflib import identify_container +from yapf.yapflib import pytree_unwrapper +from yapf.yapflib import pytree_utils +from yapf.yapflib import pytree_visitor +from yapf.yapflib import split_penalty +from yapf.yapflib import style +from yapf.yapflib import subtype_assigner + + +class YAPFTest(unittest.TestCase): + + def assertCodeEqual(self, expected_code, code): + if code != expected_code: + msg = ['Code format mismatch:', 'Expected:'] + linelen = style.Get('COLUMN_LIMIT') + for l in expected_code.splitlines(): + if len(l) > linelen: + msg.append('!> %s' % l) + else: + msg.append(' > %s' % l) + msg.append('Actual:') + for l in code.splitlines(): + if len(l) > linelen: + msg.append('!> %s' % l) + else: + msg.append(' > %s' % l) + msg.append('Diff:') + msg.extend( + difflib.unified_diff( + code.splitlines(), + expected_code.splitlines(), + fromfile='actual', + tofile='expected', + lineterm='')) + self.fail('\n'.join(msg)) + + +def ParseAndUnwrap(code, dumptree=False): + """Produces unwrapped lines from the given code. + + Parses the code into a tree, performs comment splicing and runs the + unwrapper. + + Arguments: + code: code to parse as a string + dumptree: if True, the parsed pytree (after comment splicing) is dumped + to stderr. Useful for debugging. + + Returns: + List of unwrapped lines. + """ + tree = pytree_utils.ParseCodeToTree(code) + comment_splicer.SpliceComments(tree) + continuation_splicer.SpliceContinuations(tree) + subtype_assigner.AssignSubtypes(tree) + identify_container.IdentifyContainers(tree) + split_penalty.ComputeSplitPenalties(tree) + blank_line_calculator.CalculateBlankLines(tree) + + if dumptree: + pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) + + uwlines = pytree_unwrapper.UnwrapPyTree(tree) + for uwl in uwlines: + uwl.CalculateFormattingInformation() + + return uwlines |