# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
Python parse tree definitions.
This is a very concrete parse tree; we need to keep every token and
even the comments and whitespace between tokens.
There's also a pattern matching implementation here.
__author__ = "Guido van Rossum <guido@python.org>"
from StringIO import StringIO
HUGE = 0x7FFFFFFF # maximum repeat count, default max
from .pygram import python_symbols
# printing tokens is possible but not as useful
# from .pgen2 import token // token.__dict__.items():
for name, val in python_symbols.__dict__.items():
if type(val) == int: _type_reprs[val] = name
return _type_reprs.setdefault(type_num, type_num)
Abstract base class for Node and Leaf.
This provides some default functionality and boilerplate using the
A node may be a subnode of at most one parent.
# Default values for instance variables
type = None # int: token number (< 256) or symbol number (>= 256)
parent = None # Parent node pointer, or None
children = () # Tuple of subnodes
def __new__(cls, *args, **kwds):
"""Constructor that prevents Base from being instantiated."""
assert cls is not Base, "Cannot instantiate Base"
return object.__new__(cls)
Compare two nodes for equality.
This calls the method _eq().
if self.__class__ is not other.__class__:
__hash__ = None # For Py3 compatibility.
Compare two nodes for inequality.
This calls the method _eq().
if self.__class__ is not other.__class__:
return not self._eq(other)
Compare two nodes for equality.
This is called by __eq__ and __ne__. It is only called if the two nodes
have the same type. This must be implemented by the concrete subclass.
Nodes should be considered equal if they have the same structure,
ignoring the prefix string and other context information.
raise NotImplementedError
Return a cloned (deep) copy of self.
This must be implemented by the concrete subclass.
raise NotImplementedError
Return a post-order iterator for the tree.
This must be implemented by the concrete subclass.
raise NotImplementedError
Return a pre-order iterator for the tree.
This must be implemented by the concrete subclass.
raise NotImplementedError
def set_prefix(self, prefix):
Set the prefix for the node (see Leaf class).
DEPRECATED; use the prefix property directly.
warnings.warn("set_prefix() is deprecated; use the prefix property",
DeprecationWarning, stacklevel=2)
Return the prefix for the node (see Leaf class).
DEPRECATED; use the prefix property directly.
warnings.warn("get_prefix() is deprecated; use the prefix property",
DeprecationWarning, stacklevel=2)
"""Replace this node with a new one in the parent."""
assert self.parent is not None, str(self)
if not isinstance(new, list):
for ch in self.parent.children:
assert not found, (self.parent.children, self, new)
assert found, (self.children, self, new)
self.parent.children = l_children
"""Return the line number which generated the invocant node."""
while not isinstance(node, Leaf):
Remove the node from the tree. Returns the position of the node in its
parent's children before it was removed.
for i, node in enumerate(self.parent.children):
del self.parent.children[i]
The node immediately following the invocant in their parent's children
list. If the invocant does not have a next sibling, it is None
# Can't use index(); we need to test by identity
for i, child in enumerate(self.parent.children):
return self.parent.children[i+1]
The node immediately preceding the invocant in their parent's children
list. If the invocant does not have a previous sibling, it is None.
# Can't use index(); we need to test by identity
for i, child in enumerate(self.parent.children):
return self.parent.children[i-1]
for child in self.children:
return 1 + self.parent.depth()
Return the string immediately following the invocant node. This is
effectively equivalent to node.next_sibling.prefix
next_sib = self.next_sibling
if sys.version_info < (3, 0):
return unicode(self).encode("ascii")
"""Concrete implementation for interior nodes."""
def __init__(self,type, children,
Takes a type constant (a symbol number >= 256), a sequence of
child nodes, and an optional context keyword argument.
As a side effect, the parent pointers of the children are updated.
self.children = list(children)
assert ch.parent is None, repr(ch)
self.fixers_applied = fixers_applied[:]
self.fixers_applied = None
"""Return a canonical string representation."""
return "%s(%s, %r)" % (self.__class__.__name__,
Return a pretty string representation.
This reproduces the input source exactly.
return u"".join(map(unicode, self.children))
if sys.version_info > (3, 0):
"""Compare two nodes for equality."""
return (self.type, self.children) == (other.type, other.children)
"""Return a cloned (deep) copy of self."""
return Node(self.type, [ch.clone() for ch in self.children],
fixers_applied=self.fixers_applied)
"""Return a post-order iterator for the tree."""
for child in self.children:
for node in child.post_order():
"""Return a pre-order iterator for the tree."""
for child in self.children:
for node in child.pre_order():
def _prefix_getter(self):
The whitespace and comments preceding this node in the input.
return self.children[0].prefix
def _prefix_setter(self, prefix):
self.children[0].prefix = prefix
prefix = property(_prefix_getter, _prefix_setter)
def set_child(self, i, child):
Equivalent to 'node.children[i] = child'. This method also sets the
child's parent attribute appropriately.
self.children[i].parent = None
def insert_child(self, i, child):
Equivalent to 'node.children.insert(i, child)'. This method also sets
the child's parent attribute appropriately.
self.children.insert(i, child)
def append_child(self, child):
Equivalent to 'node.children.append(child)'. This method also sets the
child's parent attribute appropriately.
self.children.append(child)
"""Concrete implementation for leaf nodes."""
# Default values for instance variables
_prefix = "" # Whitespace and comments preceding this token in the input
lineno = 0 # Line where this token starts in the input
column = 0 # Column where this token tarts in the input
def __init__(self, type, value,
Takes a type constant (a token number < 256), a string value, and an
optional context keyword argument.
assert 0 <= type < 256, type
self._prefix, (self.lineno, self.column) = context
self.fixers_applied = fixers_applied[:]
"""Return a canonical string representation."""
return "%s(%r, %r)" % (self.__class__.__name__,
Return a pretty string representation.
This reproduces the input source exactly.
return self.prefix + unicode(self.value)
if sys.version_info > (3, 0):
"""Compare two nodes for equality."""
return (self.type, self.value) == (other.type, other.value)
"""Return a cloned (deep) copy of self."""
return Leaf(self.type, self.value,
(self.prefix, (self.lineno, self.column)),
fixers_applied=self.fixers_applied)
"""Return a post-order iterator for the tree."""
"""Return a pre-order iterator for the tree."""
def _prefix_getter(self):
The whitespace and comments preceding this token in the input.
def _prefix_setter(self, prefix):
prefix = property(_prefix_getter, _prefix_setter)
def convert(gr, raw_node):
Convert raw node information to a Node or Leaf instance.
This is passed to the parser driver which calls it whenever a reduction of a
grammar rule produces a new complete node, so that the tree is build
type, value, context, children = raw_node
if children or type in gr.number2symbol:
# If there's exactly one child, return that child instead of
return Node(type, children, context=context)
return Leaf(type, value, context=context)
class BasePattern(object):
A pattern is a tree matching pattern.
It looks for a specific node type (token or symbol), and
optionally for a specific content.
This is an abstract base class. There are three concrete
- LeafPattern matches a single leaf node;
- NodePattern matches a single node (usually non-leaf);
- WildcardPattern matches a sequence of nodes of variable length.
# Defaults for instance variables
type = None # Node type (token if < 256, symbol if >= 256)
content = None # Optional content matching pattern
name = None # Optional name used to store match in results dict
def __new__(cls, *args, **kwds):
"""Constructor that prevents BasePattern from being instantiated."""
assert cls is not BasePattern, "Cannot instantiate BasePattern"
return object.__new__(cls)
args = [type_repr(self.type), self.content, self.name]
while args and args[-1] is None:
return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args)))
A subclass can define this as a hook for optimizations.
Returns either self or another node with the same effect.
def match(self, node, results=None):
Does this pattern exactly match a node?
Returns True if it matches, False if not.
If results is not None, it must be a dict which will be
updated with the nodes matching named subpatterns.
Default implementation for non-wildcard patterns.
if self.type is not None and node.type != self.type:
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
