# Reason last stmt is continued (or C_NONE if it's not).
(C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,
C_STRING_NEXT_LINES, C_BRACKET) = range(5)
if 0: # for throwaway debugging output
sys.__stdout__.write(" ".join(map(str, stuff)) + "\n")
# Find what looks like the start of a popular stmt.
_synchre = re.compile(r"""
""", re.VERBOSE | re.MULTILINE).search
# Match blank line or non-indenting comment line.
_junkre = re.compile(r"""
# Match any flavor of string; the terminating quote is optional
# so that we're robust in the face of incomplete program text.
_match_stringre = re.compile(r"""
| " [^"\\\n]* (?: \\. [^"\\\n]* )* "?
| ' [^'\\\n]* (?: \\. [^'\\\n]* )* '?
""", re.VERBOSE | re.DOTALL).match
# Match a line that starts with something interesting;
# used to find the first item of a bracket structure.
_itemre = re.compile(r"""
[^\s#\\] # if we match, m.end()-1 is the interesting char
# Match start of stmts that should be followed by a dedent.
_closere = re.compile(r"""
# Chew up non-special chars as quickly as possible. If match is
# successful, m.end() less 1 is the index of the last boring char
# matched. If match is unsuccessful, the string starts with an
_chew_ordinaryre = re.compile(r"""
# Build translation table to map uninteresting chars to "x", open
# brackets to "(", and close brackets to ")".
UnicodeType = type(unicode(""))
def __init__(self, indentwidth, tabwidth):
self.indentwidth = indentwidth
assert len(str) == 0 or str[-1] == '\n'
if type(str) is UnicodeType:
# The parse functions have no idea what to do with Unicode, so
# replace all Unicode characters with "x". This is "safe"
# so long as the only characters germane to parsing the structure
# of Python are 7-bit ASCII. It's *necessary* because Unicode
# strings don't have a .translate() method that supports
for raw in map(ord, uniphooey):
push(raw < 127 and chr(raw) or "x")
# Return index of a good place to begin parsing, as close to the
# end of the string as possible. This will be the start of some
# popular stmt like "if" or "def". Return None if none found:
# the caller should pass more prior context then, if possible, or
# if not (the entire program text up until the point of interest
# has already been tried) pass 0 to set_lo.
# This will be reliable iff given a reliable is_char_in_string
# function, meaning that when it says "no", it's absolutely
# guaranteed that the char is not in a string.
def find_good_parse_start(self, is_char_in_string=None,
str, pos = self.str, None
if not is_char_in_string:
# no clue -- make the caller pass everything
# Peek back from the end for a good place to start,
# but don't try too often; pos will be left None, or
# bumped to a legitimate synch point.
i = str.rfind(":\n", 0, limit)
i = str.rfind('\n', 0, i) + 1 # start of colon line
m = _synchre(str, i, limit)
if m and not is_char_in_string(m.start()):
# Nothing looks like a block-opener, or stuff does
# but is_char_in_string keeps returning true; most likely
# we're in or near a giant string, the colorizer hasn't
# caught up enough to be helpful, or there simply *aren't*
# any interesting stmts. In any of these cases we're
# going to have to parse the whole thing to be sure, so
# give it one last try from the start, but stop wasting
# time here regardless of the outcome.
if m and not is_char_in_string(m.start()):
# Peeking back worked; look forward until _synchre no longer
if not is_char_in_string(s):
# Throw away the start of the string. Intended to be called with
# find_good_parse_start's result.
assert lo == 0 or self.str[lo-1] == '\n'
# As quickly as humanly possible <wink>, find the line numbers (0-
# based) of the non-continuation lines.
# Creates self.{goodlines, continuation}.
if self.study_level >= 1:
# Map all uninteresting characters to "x", all open brackets
# to "(", all close brackets to ")", then collapse runs of
# uninteresting characters. This can cut the number of chars
# by a factor of 10-40, and so greatly speed the following loop.
str = str.translate(_tran)
str = str.replace('xxxxxxxx', 'x')
str = str.replace('xxxx', 'x')
str = str.replace('xx', 'x')
str = str.replace('xx', 'x')
str = str.replace('\nx', '\n')
# note that replacing x\n with \n would be incorrect, because
# x may be preceded by a backslash
# March over the squashed version of the program, accumulating
# the line numbers of non-continued stmts, and determining
# whether & why the last stmt is a continuation.
level = lno = 0 # level is nesting level; lno is line number
self.goodlines = goodlines = [0]
push_good = goodlines.append
# cases are checked in decreasing order of frequency
# else we're in an unclosed bracket structure
# else the program is invalid, but we can't complain
if ch == '"' or ch == "'":
if str[i-1:i+2] == quote * 3:
if str[i-1:i+w] == quote:
# unterminated single-quoted string
# else comment char or paren inside string
# didn't break out of the loop, so we're still
if (lno - 1) == firstlno:
# before the previous \n in str, we were in the first
continuation = C_STRING_FIRST_LINE
continuation = C_STRING_NEXT_LINES
continue # with outer loop
continuation = C_BACKSLASH
# The last stmt may be continued for all 3 reasons.
# String continuation takes precedence over bracket
# continuation, which beats backslash continuation.
if (continuation != C_STRING_FIRST_LINE
and continuation != C_STRING_NEXT_LINES and level > 0):
self.continuation = continuation
# Push the final line number as a sentinel value, regardless of
# whether it's continued.
assert (continuation == C_NONE) == (goodlines[-1] == lno)
def get_continuation_type(self):
# study1 was sufficient to determine the continuation status,
# but doing more requires looking at every character. study2
# does this for the last interesting statement in the block.
# self.stmt_start, stmt_end
# slice indices of last interesting stmt
# the bracketing structure of the last interesting stmt;
# for example, for the statement "say(boo) or die", stmt_bracketing
# will be [(0, 0), (3, 1), (8, 0)]. Strings and comments are
# treated as brackets, for the matter.
# last non-whitespace character before optional trailing
# self.lastopenbracketpos
# if continuation is C_BRACKET, index of last open bracket
if self.study_level >= 2:
# Set p and q to slice indices of last interesting stmt.
str, goodlines = self.str, self.goodlines
p = len(str) # index of newest line
# p is the index of the stmt at line number goodlines[i].
# Move p back to the stmt at line number goodlines[i-1].
for nothing in range(goodlines[i-1], goodlines[i]):
# tricky: sets p to 0 if no preceding newline
p = str.rfind('\n', 0, p-1) + 1
# The stmt str[p:q] isn't a continuation, but may be blank
# or a non-indenting comment line.
self.stmt_start, self.stmt_end = p, q
# Analyze this stmt, to find the last open bracket (if any)
# and last interesting character (if any).
stack = [] # stack of open bracket indices
push_stack = stack.append
# suck up all except ()[]{}'"#\\
m = _chew_ordinaryre(str, p, q)
# we skipped at least one boring char
# back up over totally boring whitespace
i = newp - 1 # index of last boring char
while i >= p and str[i] in " \t\n":
bracketing.append((p, len(stack)))
bracketing.append((p, len(stack)))
if ch == '"' or ch == "'":
# Note that study1 did this with a Python loop, but
# we use a regexp here; the reason is speed in both
# cases; the string may be huge, but study1 pre-squashed
# strings to a couple of characters per line. study1
# also needed to keep track of newlines, and we don't
bracketing.append((p, len(stack)+1))
p = _match_stringre(str, p, q).end()
bracketing.append((p, len(stack)))
# consume comment and trailing newline
bracketing.append((p, len(stack)+1))
p = str.find('\n', p, q) + 1
bracketing.append((p, len(stack)))
p = p+1 # beyond backslash
# the program is invalid, but can't complain
p = p+1 # beyond escaped char
self.lastopenbracketpos = stack[-1]
self.stmt_bracketing = tuple(bracketing)
# Assuming continuation is C_BRACKET, return the number
# of spaces the next line should be indented.
def compute_bracket_indent(self):
assert self.continuation == C_BRACKET
j = self.lastopenbracketpos
origi = i = str.rfind('\n', 0, j) + 1
j = j+1 # one beyond open bracket
# find first list item; set i to start of its line
j = m.end() - 1 # index of first interesting char
# this line is junk; advance to next line
i = j = str.find('\n', j) + 1
# nothing interesting follows the bracket;
# reproduce the bracket line's indentation + a level
return len(str[i:j].expandtabs(self.tabwidth)) + extra
# Return number of physical lines in last stmt (whether or not
# it's an interesting stmt! this is intended to be called when
# continuation is C_BACKSLASH).
def get_num_lines_in_stmt(self):
goodlines = self.goodlines
return goodlines[-1] - goodlines[-2]
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
