# Copyright 2006 Google, Inc. All Rights Reserved.

[0] Fix | Delete

# Licensed to PSF under a Contributor Agreement.

[1] Fix | Delete

[2] Fix | Delete

"""Pattern compiler.

[3] Fix | Delete

[4] Fix | Delete

The grammar is taken from PatternGrammar.txt.

[5] Fix | Delete

[6] Fix | Delete

The compiler compiles a pattern to a pytree.*Pattern instance.

[7] Fix | Delete

"""

[8] Fix | Delete

[9] Fix | Delete

__author__ = "Guido van Rossum <guido@python.org>"

[10] Fix | Delete

[11] Fix | Delete

# Python imports

[12] Fix | Delete

import StringIO

[13] Fix | Delete

[14] Fix | Delete

# Fairly local imports

[15] Fix | Delete

from .pgen2 import driver, literals, token, tokenize, parse, grammar

[16] Fix | Delete

[17] Fix | Delete

# Really local imports

[18] Fix | Delete

from . import pytree

[19] Fix | Delete

from . import pygram

[20] Fix | Delete

[21] Fix | Delete

[22] Fix | Delete

class PatternSyntaxError(Exception):

[23] Fix | Delete

pass

[24] Fix | Delete

[25] Fix | Delete

[26] Fix | Delete

def tokenize_wrapper(input):

[27] Fix | Delete

"""Tokenizes a string suppressing significant whitespace."""

[28] Fix | Delete

skip = set((token.NEWLINE, token.INDENT, token.DEDENT))

[29] Fix | Delete

tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)

[30] Fix | Delete

for quintuple in tokens:

[31] Fix | Delete

type, value, start, end, line_text = quintuple

[32] Fix | Delete

if type not in skip:

[33] Fix | Delete

yield quintuple

[34] Fix | Delete

[35] Fix | Delete

[36] Fix | Delete

class PatternCompiler(object):

[37] Fix | Delete

[38] Fix | Delete

def __init__(self, grammar_file=None):

[39] Fix | Delete

"""Initializer.

[40] Fix | Delete

[41] Fix | Delete

Takes an optional alternative filename for the pattern grammar.

[42] Fix | Delete

"""

[43] Fix | Delete

if grammar_file is None:

[44] Fix | Delete

self.grammar = pygram.pattern_grammar

[45] Fix | Delete

self.syms = pygram.pattern_symbols

[46] Fix | Delete

else:

[47] Fix | Delete

self.grammar = driver.load_grammar(grammar_file)

[48] Fix | Delete

self.syms = pygram.Symbols(self.grammar)

[49] Fix | Delete

self.pygrammar = pygram.python_grammar

[50] Fix | Delete

self.pysyms = pygram.python_symbols

[51] Fix | Delete

self.driver = driver.Driver(self.grammar, convert=pattern_convert)

[52] Fix | Delete

[53] Fix | Delete

def compile_pattern(self, input, debug=False, with_tree=False):

[54] Fix | Delete

"""Compiles a pattern string to a nested pytree.*Pattern object."""

[55] Fix | Delete

tokens = tokenize_wrapper(input)

[56] Fix | Delete

try:

[57] Fix | Delete

root = self.driver.parse_tokens(tokens, debug=debug)

[58] Fix | Delete

except parse.ParseError as e:

[59] Fix | Delete

raise PatternSyntaxError(str(e))

[60] Fix | Delete

if with_tree:

[61] Fix | Delete

return self.compile_node(root), root

[62] Fix | Delete

else:

[63] Fix | Delete

return self.compile_node(root)

[64] Fix | Delete

[65] Fix | Delete

def compile_node(self, node):

[66] Fix | Delete

"""Compiles a node, recursively.

[67] Fix | Delete

[68] Fix | Delete

This is one big switch on the node type.

[69] Fix | Delete

"""

[70] Fix | Delete

# XXX Optimize certain Wildcard-containing-Wildcard patterns

[71] Fix | Delete

# that can be merged

[72] Fix | Delete

if node.type == self.syms.Matcher:

[73] Fix | Delete

node = node.children[0] # Avoid unneeded recursion

[74] Fix | Delete

[75] Fix | Delete

if node.type == self.syms.Alternatives:

[76] Fix | Delete

# Skip the odd children since they are just '|' tokens

[77] Fix | Delete

alts = [self.compile_node(ch) for ch in node.children[::2]]

[78] Fix | Delete

if len(alts) == 1:

[79] Fix | Delete

return alts[0]

[80] Fix | Delete

p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1)

[81] Fix | Delete

return p.optimize()

[82] Fix | Delete

[83] Fix | Delete

if node.type == self.syms.Alternative:

[84] Fix | Delete

units = [self.compile_node(ch) for ch in node.children]

[85] Fix | Delete

if len(units) == 1:

[86] Fix | Delete

return units[0]

[87] Fix | Delete

p = pytree.WildcardPattern([units], min=1, max=1)

[88] Fix | Delete

return p.optimize()

[89] Fix | Delete

[90] Fix | Delete

if node.type == self.syms.NegatedUnit:

[91] Fix | Delete

pattern = self.compile_basic(node.children[1:])

[92] Fix | Delete

p = pytree.NegatedPattern(pattern)

[93] Fix | Delete

return p.optimize()

[94] Fix | Delete

[95] Fix | Delete

assert node.type == self.syms.Unit

[96] Fix | Delete

[97] Fix | Delete

name = None

[98] Fix | Delete

nodes = node.children

[99] Fix | Delete

if len(nodes) >= 3 and nodes[1].type == token.EQUAL:

[100] Fix | Delete

name = nodes[0].value

[101] Fix | Delete

nodes = nodes[2:]

[102] Fix | Delete

repeat = None

[103] Fix | Delete

if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater:

[104] Fix | Delete

repeat = nodes[-1]

[105] Fix | Delete

nodes = nodes[:-1]

[106] Fix | Delete

[107] Fix | Delete

# Now we've reduced it to: STRING | NAME [Details] | (...) | [...]

[108] Fix | Delete

pattern = self.compile_basic(nodes, repeat)

[109] Fix | Delete

[110] Fix | Delete

if repeat is not None:

[111] Fix | Delete

assert repeat.type == self.syms.Repeater

[112] Fix | Delete

children = repeat.children

[113] Fix | Delete

child = children[0]

[114] Fix | Delete

if child.type == token.STAR:

[115] Fix | Delete

min = 0

[116] Fix | Delete

max = pytree.HUGE

[117] Fix | Delete

elif child.type == token.PLUS:

[118] Fix | Delete

min = 1

[119] Fix | Delete

max = pytree.HUGE

[120] Fix | Delete

elif child.type == token.LBRACE:

[121] Fix | Delete

assert children[-1].type == token.RBRACE

[122] Fix | Delete

assert len(children) in (3, 5)

[123] Fix | Delete

min = max = self.get_int(children[1])

[124] Fix | Delete

if len(children) == 5:

[125] Fix | Delete

max = self.get_int(children[3])

[126] Fix | Delete

else:

[127] Fix | Delete

assert False

[128] Fix | Delete

if min != 1 or max != 1:

[129] Fix | Delete

pattern = pattern.optimize()

[130] Fix | Delete

pattern = pytree.WildcardPattern([[pattern]], min=min, max=max)

[131] Fix | Delete

[132] Fix | Delete

if name is not None:

[133] Fix | Delete

pattern.name = name

[134] Fix | Delete

return pattern.optimize()

[135] Fix | Delete

[136] Fix | Delete

def compile_basic(self, nodes, repeat=None):

[137] Fix | Delete

# Compile STRING | NAME [Details] | (...) | [...]

[138] Fix | Delete

assert len(nodes) >= 1

[139] Fix | Delete

node = nodes[0]

[140] Fix | Delete

if node.type == token.STRING:

[141] Fix | Delete

value = unicode(literals.evalString(node.value))

[142] Fix | Delete

return pytree.LeafPattern(_type_of_literal(value), value)

[143] Fix | Delete

elif node.type == token.NAME:

[144] Fix | Delete

value = node.value

[145] Fix | Delete

if value.isupper():

[146] Fix | Delete

if value not in TOKEN_MAP:

[147] Fix | Delete

raise PatternSyntaxError("Invalid token: %r" % value)

[148] Fix | Delete

if nodes[1:]:

[149] Fix | Delete

raise PatternSyntaxError("Can't have details for token")

[150] Fix | Delete

return pytree.LeafPattern(TOKEN_MAP[value])

[151] Fix | Delete

else:

[152] Fix | Delete

if value == "any":

[153] Fix | Delete

type = None

[154] Fix | Delete

elif not value.startswith("_"):

[155] Fix | Delete

type = getattr(self.pysyms, value, None)

[156] Fix | Delete

if type is None:

[157] Fix | Delete

raise PatternSyntaxError("Invalid symbol: %r" % value)

[158] Fix | Delete

if nodes[1:]: # Details present

[159] Fix | Delete

content = [self.compile_node(nodes[1].children[1])]

[160] Fix | Delete

else:

[161] Fix | Delete

content = None

[162] Fix | Delete

return pytree.NodePattern(type, content)

[163] Fix | Delete

elif node.value == "(":

[164] Fix | Delete

return self.compile_node(nodes[1])

[165] Fix | Delete

elif node.value == "[":

[166] Fix | Delete

assert repeat is None

[167] Fix | Delete

subpattern = self.compile_node(nodes[1])

[168] Fix | Delete

return pytree.WildcardPattern([[subpattern]], min=0, max=1)

[169] Fix | Delete

assert False, node

[170] Fix | Delete

[171] Fix | Delete

def get_int(self, node):

[172] Fix | Delete

assert node.type == token.NUMBER

[173] Fix | Delete

return int(node.value)

[174] Fix | Delete

[175] Fix | Delete

[176] Fix | Delete

# Map named tokens to the type value for a LeafPattern

[177] Fix | Delete

TOKEN_MAP = {"NAME": token.NAME,

[178] Fix | Delete

"STRING": token.STRING,

[179] Fix | Delete

"NUMBER": token.NUMBER,

[180] Fix | Delete

"TOKEN": None}

[181] Fix | Delete

[182] Fix | Delete

[183] Fix | Delete

def _type_of_literal(value):

[184] Fix | Delete

if value[0].isalpha():

[185] Fix | Delete

return token.NAME

[186] Fix | Delete

elif value in grammar.opmap:

[187] Fix | Delete

return grammar.opmap[value]

[188] Fix | Delete

else:

[189] Fix | Delete

return None

[190] Fix | Delete

[191] Fix | Delete

[192] Fix | Delete

def pattern_convert(grammar, raw_node_info):

[193] Fix | Delete

"""Converts raw node information to a Node or Leaf instance."""

[194] Fix | Delete

type, value, context, children = raw_node_info

[195] Fix | Delete

if children or type in grammar.number2symbol:

[196] Fix | Delete

return pytree.Node(type, children, context=context)

[197] Fix | Delete

else:

[198] Fix | Delete

return pytree.Leaf(type, value, context=context)

[199] Fix | Delete

[200] Fix | Delete

[201] Fix | Delete

def compile_pattern(pattern):

[202] Fix | Delete

return PatternCompiler().compile_pattern(pattern)

[203] Fix | Delete

[204] Fix | Delete