def __repr__(self):

annotated_type = '%d (%s)' % (self.type, tok_name[self.type])

return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %

self._replace(type=annotated_type))

@property

def exact_type(self):

if self.type == OP and self.string in EXACT_TOKEN_TYPES:

return EXACT_TOKEN_TYPES[self.string]

else:

# The valid string prefixes. Only contain the lower case versions,

# and don't contain any permutations (include 'fr', but not

# 'rf'). The various permutations will be generated.

_valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr']

# if we add binary f-strings, add: ['fb', 'fbr']

result = {''}

for prefix in _valid_string_prefixes:

for t in _itertools.permutations(prefix):

# create a list with upper and lower versions of each

# character

for u in _itertools.product(*[(c, c.upper()) for c in t]):

result.add(''.join(u))

def __init__(self):

self.tokens = []

self.prev_row = 1

self.prev_col = 0

self.encoding = None

def add_whitespace(self, start):

row, col = start

if row < self.prev_row or row == self.prev_row and col < self.prev_col:

raise ValueError("start ({},{}) precedes previous end ({},{})"

.format(row, col, self.prev_row, self.prev_col))

row_offset = row - self.prev_row

if row_offset:

self.tokens.append("\\\n" * row_offset)

self.prev_col = 0

col_offset = col - self.prev_col

if col_offset:

self.tokens.append(" " * col_offset)

def untokenize(self, iterable):

it = iter(iterable)

indents = []

startline = False

for t in it:

if len(t) == 2:

self.compat(t, it)

break

tok_type, token, start, end, line = t

if tok_type == ENCODING:

self.encoding = token

continue

if tok_type == ENDMARKER:

break

if tok_type == INDENT:

indents.append(token)

continue

elif tok_type == DEDENT:

indents.pop()

self.prev_row, self.prev_col = end

continue

elif tok_type in (NEWLINE, NL):

startline = True

elif startline and indents:

indent = indents[-1]

if start[1] >= len(indent):

self.tokens.append(indent)

self.prev_col = len(indent)

startline = False

self.add_whitespace(start)

self.tokens.append(token)

self.prev_row, self.prev_col = end

if tok_type in (NEWLINE, NL):

self.prev_row += 1

self.prev_col = 0

return "".join(self.tokens)

def compat(self, token, iterable):

indents = []

toks_append = self.tokens.append

startline = token[0] in (NEWLINE, NL)

prevstring = False

for tok in _itertools.chain([token], iterable):

toknum, tokval = tok[:2]

if toknum == ENCODING:

self.encoding = tokval

continue

if toknum in (NAME, NUMBER):

tokval += ' '

# Insert a space between two consecutive strings

if toknum == STRING:

if prevstring:

tokval = ' ' + tokval

prevstring = True

else:

prevstring = False

if toknum == INDENT:

indents.append(tokval)

continue

elif toknum == DEDENT:

indents.pop()

continue

elif toknum in (NEWLINE, NL):

startline = True

elif startline and indents:

toks_append(indents[-1])

startline = False

"""Transform tokens back into Python source code.

ut = Untokenizer()

out = ut.untokenize(iterable)

if ut.encoding is not None:

out = out.encode(ut.encoding)

"""Imitates get_normal_name in tokenizer.c."""

# Only care about the first 12 characters.

enc = orig_enc[:12].lower().replace("_", "-")

if enc == "utf-8" or enc.startswith("utf-8-"):

return "utf-8"

if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \

enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):

return "iso-8859-1"

"""

try:

filename = readline.__self__.name

except AttributeError:

filename = None

bom_found = False

encoding = None

default = 'utf-8'

def read_or_stop():

try:

return readline()

except StopIteration:

return b''

def find_cookie(line):

try:

# Decode as UTF-8. Either the line is an encoding declaration,

# in which case it should be pure ASCII, or it must be UTF-8

# per default encoding.

line_string = line.decode('utf-8')

except UnicodeDecodeError:

msg = "invalid or missing encoding declaration"

if filename is not None:

msg = '{} for {!r}'.format(msg, filename)

raise SyntaxError(msg)

match = cookie_re.match(line_string)

if not match:

return None

encoding = _get_normal_name(match.group(1))

try:

codec = lookup(encoding)

except LookupError:

# This behaviour mimics the Python interpreter

if filename is None:

msg = "unknown encoding: " + encoding

else:

msg = "unknown encoding for {!r}: {}".format(filename,

encoding)

raise SyntaxError(msg)

if bom_found:

if encoding != 'utf-8':

# This behaviour mimics the Python interpreter

if filename is None:

msg = 'encoding problem: utf-8'

else:

msg = 'encoding problem for {!r}: utf-8'.format(filename)

raise SyntaxError(msg)

encoding += '-sig'

return encoding

first = read_or_stop()

if first.startswith(BOM_UTF8):

bom_found = True

first = first[3:]

default = 'utf-8-sig'

if not first:

return default, []

encoding = find_cookie(first)

if encoding:

return encoding, [first]

if not blank_re.match(first):

return default, [first]

second = read_or_stop()

if not second:

return default, [first]

encoding = find_cookie(second)

if encoding:

return encoding, [first, second]

"""Open a file in read only mode using the encoding detected by

buffer = _builtin_open(filename, 'rb')

try:

encoding, lines = detect_encoding(buffer.readline)

buffer.seek(0)

text = TextIOWrapper(buffer, encoding, line_buffering=True)

text.mode = 'r'

return text

except:

buffer.close()

"""

encoding, consumed = detect_encoding(readline)

empty = _itertools.repeat(b"")

rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)

lnum = parenlev = continued = 0

numchars = '0123456789'

contstr, needcont = '', 0

contline = None

indents = [0]

if encoding is not None:

if encoding == "utf-8-sig":

# BOM will already have been stripped.

encoding = "utf-8"

yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')

last_line = b''

line = b''

while True: # loop over lines in stream

try:

# We capture the value of the line variable here because

# readline uses the empty string '' to signal end of input,

# hence `line` itself will always be overwritten at the end

# of this loop.

last_line = line

line = readline()

except StopIteration:

line = b''

if encoding is not None:

line = line.decode(encoding)

lnum += 1

pos, max = 0, len(line)

if contstr: # continued string

if not line:

raise TokenError("EOF in multi-line string", strstart)

endmatch = endprog.match(line)

if endmatch:

pos = end = endmatch.end(0)

yield TokenInfo(STRING, contstr + line[:end],

strstart, (lnum, end), contline + line)

contstr, needcont = '', 0

contline = None

elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':

yield TokenInfo(ERRORTOKEN, contstr + line,

strstart, (lnum, len(line)), contline)

contstr = ''

contline = None

continue

else:

contstr = contstr + line

contline = contline + line

continue

elif parenlev == 0 and not continued: # new statement

if not line: break

column = 0

while pos < max: # measure leading whitespace

if line[pos] == ' ':

column += 1

elif line[pos] == '\t':

column = (column//tabsize + 1)*tabsize

elif line[pos] == '\f':

column = 0

else:

break

pos += 1

if pos == max:

break

if line[pos] in '#\r\n': # skip comments or blank lines

if line[pos] == '#':

comment_token = line[pos:].rstrip('\r\n')

yield TokenInfo(COMMENT, comment_token,

(lnum, pos), (lnum, pos + len(comment_token)), line)

pos += len(comment_token)

yield TokenInfo(NL, line[pos:],

(lnum, pos), (lnum, len(line)), line)

continue

if column > indents[-1]: # count indents or dedents

indents.append(column)

yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)

while column < indents[-1]:

if column not in indents:

raise IndentationError(

"unindent does not match any outer indentation level",

("<tokenize>", lnum, pos, line))

indents = indents[:-1]

yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)

else: # continued statement

if not line:

raise TokenError("EOF in multi-line statement", (lnum, 0))

continued = 0

while pos < max:

pseudomatch = _compile(PseudoToken).match(line, pos)

if pseudomatch: # scan for tokens

start, end = pseudomatch.span(1)

spos, epos, pos = (lnum, start), (lnum, end), end

if start == end:

continue

token, initial = line[start:end], line[start]

if (initial in numchars or # ordinary number

(initial == '.' and token != '.' and token != '...')):

yield TokenInfo(NUMBER, token, spos, epos, line)

elif initial in '\r\n':

if parenlev > 0:

yield TokenInfo(NL, token, spos, epos, line)

else:

yield TokenInfo(NEWLINE, token, spos, epos, line)

elif initial == '#':

assert not token.endswith("\n")

yield TokenInfo(COMMENT, token, spos, epos, line)

elif token in triple_quoted:

endprog = _compile(endpats[token])

endmatch = endprog.match(line, pos)

if endmatch: # all on one line

pos = endmatch.end(0)

token = line[start:pos]

yield TokenInfo(STRING, token, spos, (lnum, pos), line)

else:

strstart = (lnum, start) # multiple lines

contstr = line[start:]

contline = line

break

# Check up to the first 3 chars of the token to see if

# they're in the single_quoted set. If so, they start

# a string.

# We're using the first 3, because we're looking for

# "rb'" (for example) at the start of the token. If

# we switch to longer prefixes, this needs to be

# adjusted.

# Note that initial == token[:1].

# Also note that single quote checking must come after

# triple quote checking (above).

elif (initial in single_quoted or

token[:2] in single_quoted or

token[:3] in single_quoted):

if token[-1] == '\n': # continued string

strstart = (lnum, start)

# Again, using the first 3 chars of the

# token. This is looking for the matching end

# regex for the correct type of quote

# character. So it's really looking for

# endpats["'"] or endpats['"'], by trying to

# skip string prefix characters, if any.

endprog = _compile(endpats.get(initial) or

endpats.get(token[1]) or

endpats.get(token[2]))

contstr, needcont = line[start:], 1

contline = line

break

else: # ordinary string

yield TokenInfo(STRING, token, spos, epos, line)

elif initial.isidentifier(): # ordinary name

yield TokenInfo(NAME, token, spos, epos, line)

elif initial == '\\': # continued stmt

continued = 1

else:

if initial in '([{':

parenlev += 1

elif initial in ')]}':

parenlev -= 1

yield TokenInfo(OP, token, spos, epos, line)

else:

yield TokenInfo(ERRORTOKEN, line[pos],

(lnum, pos), (lnum, pos+1), line)

pos += 1

# Add an implicit NEWLINE if the input doesn't end in one

if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):

yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')

for indent in indents[1:]: # pop remaining indent levels

yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')

"""Tokenize a source reading Python code as unicode strings.

import argparse

# Helper error handling routines

def perror(message):

sys.stderr.write(message)

sys.stderr.write('\n')

def error(message, filename=None, location=None):

if location:

args = (filename,) + location + (message,)

perror("%s:%d:%d: error: %s" % args)

elif filename:

perror("%s: error: %s" % (filename, message))

else:

perror("error: %s" % message)

sys.exit(1)

# Parse the arguments and options

parser = argparse.ArgumentParser(prog='python -m tokenize')

parser.add_argument(dest='filename', nargs='?',

metavar='filename.py',

help='the file to tokenize; defaults to stdin')

parser.add_argument('-e', '--exact', dest='exact', action='store_true',

help='display token names using the exact type')

args = parser.parse_args()

try:

# Tokenize the input

if args.filename:

filename = args.filename

with _builtin_open(filename, 'rb') as f:

tokens = list(tokenize(f.readline))

else:

filename = "<stdin>"

tokens = _tokenize(sys.stdin.readline, None)

# Output the tokenization

for token in tokens:

token_type = token.type

if args.exact:

token_type = token.exact_type

token_range = "%d,%d-%d,%d:" % (token.start + token.end)

print("%-20s%-15s%-15r" %

(token_range, tok_name[token_type], token.string))

except IndentationError as err:

line, column = err.args[1][1:3]

error(err.args[0], filename, (line, column))

except TokenError as err:

line, column = err.args[1]

error(err.args[0], filename, (line, column))

except SyntaxError as err:

error(err, filename)

except OSError as err:

error(err)

except KeyboardInterrupt:

print("interrupted\n")

except Exception as err:

perror("unexpected error: %s" % err)