diff options
Diffstat (limited to 'yapf/yapflib/subtype_assigner.py')
| -rw-r--r-- | yapf/yapflib/subtype_assigner.py | 253 |
1 files changed, 160 insertions, 93 deletions
diff --git a/yapf/yapflib/subtype_assigner.py b/yapf/yapflib/subtype_assigner.py index 8bf3d8d..7b45586 100644 --- a/yapf/yapflib/subtype_assigner.py +++ b/yapf/yapflib/subtype_assigner.py @@ -11,28 +11,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Subtype assigner for lib2to3 trees. +"""Subtype assigner for format tokens. -This module assigns extra type information to the lib2to3 trees. This -information is more specific than whether something is an operator or an -identifier. For instance, it can specify if a node in the tree is part of a -subscript. +This module assigns extra type information to format tokens. This information is +more specific than whether something is an operator or an identifier. For +instance, it can specify if a node in the tree is part of a subscript. AssignSubtypes(): the main function exported by this module. Annotations: - subtype: The subtype of a pytree token. See 'format_token' module for a list - of subtypes. + subtype: The subtype of a pytree token. See 'subtypes' module for a list of + subtypes. """ from lib2to3 import pytree -from lib2to3.pgen2 import token +from lib2to3.pgen2 import token as grammar_token from lib2to3.pygram import python_symbols as syms from yapf.yapflib import format_token from yapf.yapflib import pytree_utils from yapf.yapflib import pytree_visitor from yapf.yapflib import style +from yapf.yapflib import subtypes def AssignSubtypes(tree): @@ -47,10 +47,10 @@ def AssignSubtypes(tree): # Map tokens in argument lists to their respective subtype. _ARGLIST_TOKEN_TO_SUBTYPE = { - '=': format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - ':': format_token.Subtype.TYPED_NAME, - '*': format_token.Subtype.VARARGS_STAR, - '**': format_token.Subtype.KWARGS_STAR_STAR, + '=': subtypes.DEFAULT_OR_NAMED_ASSIGN, + ':': subtypes.TYPED_NAME, + '*': subtypes.VARARGS_STAR, + '**': subtypes.KWARGS_STAR_STAR, } @@ -73,35 +73,31 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): for child in node.children: if pytree_utils.NodeName(child) == 'comp_for': comp_for = True - _AppendFirstLeafTokenSubtype(child, - format_token.Subtype.DICT_SET_GENERATOR) - elif pytree_utils.NodeName(child) in ('COLON', 'DOUBLESTAR'): + _AppendFirstLeafTokenSubtype(child, subtypes.DICT_SET_GENERATOR) + elif child.type in (grammar_token.COLON, grammar_token.DOUBLESTAR): dict_maker = True if not comp_for and dict_maker: last_was_colon = False unpacking = False for child in node.children: - if pytree_utils.NodeName(child) == 'DOUBLESTAR': - _AppendFirstLeafTokenSubtype(child, - format_token.Subtype.KWARGS_STAR_STAR) + if child.type == grammar_token.DOUBLESTAR: + _AppendFirstLeafTokenSubtype(child, subtypes.KWARGS_STAR_STAR) if last_was_colon: if style.Get('INDENT_DICTIONARY_VALUE'): _InsertPseudoParentheses(child) else: - _AppendFirstLeafTokenSubtype(child, - format_token.Subtype.DICTIONARY_VALUE) + _AppendFirstLeafTokenSubtype(child, subtypes.DICTIONARY_VALUE) elif (isinstance(child, pytree.Node) or (not child.value.startswith('#') and child.value not in '{:,')): # Mark the first leaf of a key entry as a DICTIONARY_KEY. We # normally want to split before them if the dictionary cannot exist # on a single line. if not unpacking or pytree_utils.FirstLeafNode(child).value == '**': - _AppendFirstLeafTokenSubtype(child, - format_token.Subtype.DICTIONARY_KEY) - _AppendSubtypeRec(child, format_token.Subtype.DICTIONARY_KEY_PART) - last_was_colon = pytree_utils.NodeName(child) == 'COLON' - if pytree_utils.NodeName(child) == 'DOUBLESTAR': + _AppendFirstLeafTokenSubtype(child, subtypes.DICTIONARY_KEY) + _AppendSubtypeRec(child, subtypes.DICTIONARY_KEY_PART) + last_was_colon = child.type == grammar_token.COLON + if child.type == grammar_token.DOUBLESTAR: unpacking = True elif last_was_colon: unpacking = False @@ -112,28 +108,28 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '=': - _AppendTokenSubtype(child, format_token.Subtype.ASSIGN_OPERATOR) + _AppendTokenSubtype(child, subtypes.ASSIGN_OPERATOR) def Visit_or_test(self, node): # pylint: disable=invalid-name # or_test ::= and_test ('or' and_test)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == 'or': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_and_test(self, node): # pylint: disable=invalid-name # and_test ::= not_test ('and' not_test)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == 'and': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_not_test(self, node): # pylint: disable=invalid-name # not_test ::= 'not' not_test | comparison for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == 'not': - _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) def Visit_comparison(self, node): # pylint: disable=invalid-name # comparison ::= expr (comp_op expr)* @@ -142,95 +138,104 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): self.Visit(child) if (isinstance(child, pytree.Leaf) and child.value in {'<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is'}): - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) elif pytree_utils.NodeName(child) == 'comp_op': for grandchild in child.children: - _AppendTokenSubtype(grandchild, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(grandchild, subtypes.BINARY_OPERATOR) def Visit_star_expr(self, node): # pylint: disable=invalid-name # star_expr ::= '*' expr for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '*': - _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) - _AppendTokenSubtype(child, format_token.Subtype.VARARGS_STAR) + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.VARARGS_STAR) def Visit_expr(self, node): # pylint: disable=invalid-name # expr ::= xor_expr ('|' xor_expr)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '|': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_xor_expr(self, node): # pylint: disable=invalid-name # xor_expr ::= and_expr ('^' and_expr)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '^': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_and_expr(self, node): # pylint: disable=invalid-name # and_expr ::= shift_expr ('&' shift_expr)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '&': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_shift_expr(self, node): # pylint: disable=invalid-name # shift_expr ::= arith_expr (('<<'|'>>') arith_expr)* for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value in {'<<', '>>'}: - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_arith_expr(self, node): # pylint: disable=invalid-name # arith_expr ::= term (('+'|'-') term)* for child in node.children: self.Visit(child) - if isinstance(child, pytree.Leaf) and child.value in '+-': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + if _IsAExprOperator(child): + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + if _IsSimpleExpression(node): + for child in node.children: + if _IsAExprOperator(child): + _AppendTokenSubtype(child, subtypes.SIMPLE_EXPRESSION) def Visit_term(self, node): # pylint: disable=invalid-name - # term ::= factor (('*'|'/'|'%'|'//') factor)* + # term ::= factor (('*'|'/'|'%'|'//'|'@') factor)* for child in node.children: self.Visit(child) - if (isinstance(child, pytree.Leaf) and - child.value in {'*', '/', '%', '//'}): - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + if _IsMExprOperator(child): + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) + + if _IsSimpleExpression(node): + for child in node.children: + if _IsMExprOperator(child): + _AppendTokenSubtype(child, subtypes.SIMPLE_EXPRESSION) def Visit_factor(self, node): # pylint: disable=invalid-name # factor ::= ('+'|'-'|'~') factor | power for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value in '+-~': - _AppendTokenSubtype(child, format_token.Subtype.UNARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.UNARY_OPERATOR) def Visit_power(self, node): # pylint: disable=invalid-name # power ::= atom trailer* ['**' factor] for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '**': - _AppendTokenSubtype(child, format_token.Subtype.BINARY_OPERATOR) + _AppendTokenSubtype(child, subtypes.BINARY_OPERATOR) def Visit_trailer(self, node): # pylint: disable=invalid-name for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value in '[]': - _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_BRACKET) + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_BRACKET) def Visit_subscript(self, node): # pylint: disable=invalid-name # subscript ::= test | [test] ':' [test] [sliceop] for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == ':': - _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_COLON) def Visit_sliceop(self, node): # pylint: disable=invalid-name # sliceop ::= ':' [test] for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == ':': - _AppendTokenSubtype(child, format_token.Subtype.SUBSCRIPT_COLON) + _AppendTokenSubtype(child, subtypes.SUBSCRIPT_COLON) def Visit_argument(self, node): # pylint: disable=invalid-name # argument ::= @@ -243,32 +248,39 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): # | '*' test (',' argument)* [',' '**' test] # | '**' test) self._ProcessArgLists(node) - _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) def Visit_tname(self, node): # pylint: disable=invalid-name self._ProcessArgLists(node) - _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) def Visit_decorator(self, node): # pylint: disable=invalid-name # decorator ::= # '@' dotted_name [ '(' [arglist] ')' ] NEWLINE for child in node.children: if isinstance(child, pytree.Leaf) and child.value == '@': - _AppendTokenSubtype(child, subtype=format_token.Subtype.DECORATOR) + _AppendTokenSubtype(child, subtype=subtypes.DECORATOR) self.Visit(child) def Visit_funcdef(self, node): # pylint: disable=invalid-name # funcdef ::= # 'def' NAME parameters ['->' test] ':' suite for child in node.children: - if pytree_utils.NodeName(child) == 'NAME' and child.value != 'def': - _AppendTokenSubtype(child, format_token.Subtype.FUNC_DEF) + if child.type == grammar_token.NAME and child.value != 'def': + _AppendTokenSubtype(child, subtypes.FUNC_DEF) break for child in node.children: self.Visit(child) + def Visit_parameters(self, node): # pylint: disable=invalid-name + # parameters ::= '(' [typedargslist] ')' + self._ProcessArgLists(node) + if len(node.children) > 2: + _AppendFirstLeafTokenSubtype(node.children[1], subtypes.PARAMETER_START) + _AppendLastLeafTokenSubtype(node.children[-2], subtypes.PARAMETER_STOP) + def Visit_typedargslist(self, node): # pylint: disable=invalid-name # typedargslist ::= # ((tfpdef ['=' test] ',')* @@ -276,20 +288,32 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): # | '**' tname) # | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) self._ProcessArgLists(node) - _SetArgListSubtype(node, format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN, - format_token.Subtype.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) + _SetArgListSubtype(node, subtypes.DEFAULT_OR_NAMED_ASSIGN, + subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST) tname = False - for child in node.children: + if not node.children: + return + + _AppendFirstLeafTokenSubtype(node.children[0], subtypes.PARAMETER_START) + _AppendLastLeafTokenSubtype(node.children[-1], subtypes.PARAMETER_STOP) + + tname = pytree_utils.NodeName(node.children[0]) == 'tname' + for i in range(1, len(node.children)): + prev_child = node.children[i - 1] + child = node.children[i] + if prev_child.type == grammar_token.COMMA: + _AppendFirstLeafTokenSubtype(child, subtypes.PARAMETER_START) + elif child.type == grammar_token.COMMA: + _AppendLastLeafTokenSubtype(prev_child, subtypes.PARAMETER_STOP) + if pytree_utils.NodeName(child) == 'tname': tname = True - _SetArgListSubtype(child, format_token.Subtype.TYPED_NAME, - format_token.Subtype.TYPED_NAME_ARG_LIST) - if not isinstance(child, pytree.Leaf): - continue - if child.value == ',': + _SetArgListSubtype(child, subtypes.TYPED_NAME, + subtypes.TYPED_NAME_ARG_LIST) + elif child.type == grammar_token.COMMA: tname = False - elif child.value == '=' and tname: - _AppendTokenSubtype(child, subtype=format_token.Subtype.TYPED_NAME) + elif child.type == grammar_token.EQUAL and tname: + _AppendTokenSubtype(child, subtype=subtypes.TYPED_NAME) tname = False def Visit_varargslist(self, node): # pylint: disable=invalid-name @@ -302,24 +326,32 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): for child in node.children: self.Visit(child) if isinstance(child, pytree.Leaf) and child.value == '=': - _AppendTokenSubtype(child, format_token.Subtype.VARARGS_LIST) + _AppendTokenSubtype(child, subtypes.VARARGS_LIST) def Visit_comp_for(self, node): # pylint: disable=invalid-name # comp_for ::= 'for' exprlist 'in' testlist_safe [comp_iter] - _AppendSubtypeRec(node, format_token.Subtype.COMP_FOR) + _AppendSubtypeRec(node, subtypes.COMP_FOR) # Mark the previous node as COMP_EXPR unless this is a nested comprehension # as these will have the outer comprehension as their previous node. attr = pytree_utils.GetNodeAnnotation(node.parent, pytree_utils.Annotation.SUBTYPE) - if not attr or format_token.Subtype.COMP_FOR not in attr: - _AppendSubtypeRec(node.parent.children[0], format_token.Subtype.COMP_EXPR) + if not attr or subtypes.COMP_FOR not in attr: + _AppendSubtypeRec(node.parent.children[0], subtypes.COMP_EXPR) self.DefaultNodeVisit(node) + def Visit_old_comp_for(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_for(node) + def Visit_comp_if(self, node): # pylint: disable=invalid-name # comp_if ::= 'if' old_test [comp_iter] - _AppendSubtypeRec(node, format_token.Subtype.COMP_IF) + _AppendSubtypeRec(node, subtypes.COMP_IF) self.DefaultNodeVisit(node) + def Visit_old_comp_if(self, node): # pylint: disable=invalid-name + # Python 3.7 + self.Visit_comp_if(node) + def _ProcessArgLists(self, node): """Common method for processing argument lists.""" for child in node.children: @@ -327,8 +359,7 @@ class _SubtypeAssigner(pytree_visitor.PyTreeVisitor): if isinstance(child, pytree.Leaf): _AppendTokenSubtype( child, - subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, - format_token.Subtype.NONE)) + subtype=_ARGLIST_TOKEN_TO_SUBTYPE.get(child.value, subtypes.NONE)) def _SetArgListSubtype(node, node_subtype, list_subtype): @@ -337,20 +368,24 @@ def _SetArgListSubtype(node, node_subtype, list_subtype): def HasSubtype(node): """Return True if the arg list has a named assign subtype.""" if isinstance(node, pytree.Leaf): - if node_subtype in pytree_utils.GetNodeAnnotation( - node, pytree_utils.Annotation.SUBTYPE, set()): - return True - return False - has_subtype = False - for child in node.children: - if pytree_utils.NodeName(child) != 'arglist': - has_subtype |= HasSubtype(child) - return has_subtype + return node_subtype in pytree_utils.GetNodeAnnotation( + node, pytree_utils.Annotation.SUBTYPE, set()) - if HasSubtype(node): for child in node.children: - if pytree_utils.NodeName(child) != 'COMMA': - _AppendFirstLeafTokenSubtype(child, list_subtype) + node_name = pytree_utils.NodeName(child) + if node_name not in {'atom', 'arglist', 'power'}: + if HasSubtype(child): + return True + + return False + + if not HasSubtype(node): + return + + for child in node.children: + node_name = pytree_utils.NodeName(child) + if node_name not in {'atom', 'COMMA'}: + _AppendFirstLeafTokenSubtype(child, list_subtype) def _AppendTokenSubtype(node, subtype): @@ -367,6 +402,14 @@ def _AppendFirstLeafTokenSubtype(node, subtype): _AppendFirstLeafTokenSubtype(node.children[0], subtype) +def _AppendLastLeafTokenSubtype(node, subtype): + """Append the last leaf token's subtypes.""" + if isinstance(node, pytree.Leaf): + _AppendTokenSubtype(node, subtype) + return + _AppendLastLeafTokenSubtype(node.children[-1], subtype) + + def _AppendSubtypeRec(node, subtype, force=True): """Append the leafs in the node to the given subtype.""" if isinstance(node, pytree.Leaf): @@ -380,18 +423,24 @@ def _InsertPseudoParentheses(node): """Insert pseudo parentheses so that dicts can be formatted correctly.""" comment_node = None if isinstance(node, pytree.Node): - if node.children[-1].type == token.COMMENT: + if node.children[-1].type == grammar_token.COMMENT: comment_node = node.children[-1].clone() node.children[-1].remove() first = pytree_utils.FirstLeafNode(node) last = pytree_utils.LastLeafNode(node) - if first == last and first.type == token.COMMENT: + if first == last and first.type == grammar_token.COMMENT: # A comment was inserted before the value, which is a pytree.Leaf. # Encompass the dictionary's value into an ATOM node. last = first.next_sibling - new_node = pytree.Node(syms.atom, [first.clone(), last.clone()]) + last_clone = last.clone() + new_node = pytree.Node(syms.atom, [first.clone(), last_clone]) + for orig_leaf, clone_leaf in zip(last.leaves(), last_clone.leaves()): + pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf) + if hasattr(orig_leaf, 'is_pseudo'): + clone_leaf.is_pseudo = orig_leaf.is_pseudo + node.replace(new_node) node = new_node last.remove() @@ -400,17 +449,19 @@ def _InsertPseudoParentheses(node): last = pytree_utils.LastLeafNode(node) lparen = pytree.Leaf( - token.LPAR, u'(', context=('', (first.get_lineno(), first.column - 1))) + grammar_token.LPAR, + u'(', + context=('', (first.get_lineno(), first.column - 1))) last_lineno = last.get_lineno() - if last.type == token.STRING and '\n' in last.value: + if last.type == grammar_token.STRING and '\n' in last.value: last_lineno += last.value.count('\n') - if last.type == token.STRING and '\n' in last.value: + if last.type == grammar_token.STRING and '\n' in last.value: last_column = len(last.value.split('\n')[-1]) + 1 else: last_column = last.column + len(last.value) + 1 rparen = pytree.Leaf( - token.RPAR, u')', context=('', (last_lineno, last_column))) + grammar_token.RPAR, u')', context=('', (last_lineno, last_column))) lparen.is_pseudo = True rparen.is_pseudo = True @@ -420,9 +471,25 @@ def _InsertPseudoParentheses(node): node.append_child(rparen) if comment_node: node.append_child(comment_node) - _AppendFirstLeafTokenSubtype(node, format_token.Subtype.DICTIONARY_VALUE) + _AppendFirstLeafTokenSubtype(node, subtypes.DICTIONARY_VALUE) else: clone = node.clone() + for orig_leaf, clone_leaf in zip(node.leaves(), clone.leaves()): + pytree_utils.CopyYapfAnnotations(orig_leaf, clone_leaf) new_node = pytree.Node(syms.atom, [lparen, clone, rparen]) node.replace(new_node) - _AppendFirstLeafTokenSubtype(clone, format_token.Subtype.DICTIONARY_VALUE) + _AppendFirstLeafTokenSubtype(clone, subtypes.DICTIONARY_VALUE) + + +def _IsAExprOperator(node): + return isinstance(node, pytree.Leaf) and node.value in {'+', '-'} + + +def _IsMExprOperator(node): + return isinstance(node, + pytree.Leaf) and node.value in {'*', '/', '%', '//', '@'} + + +def _IsSimpleExpression(node): + """A node with only leafs as children.""" + return all(isinstance(child, pytree.Leaf) for child in node.children) |