856 lines
28 KiB
Python
856 lines
28 KiB
Python
"""Python code parser."""
|
|
|
|
import sys
|
|
import textwrap
|
|
import tokenize as tk
|
|
from io import StringIO
|
|
from itertools import chain, dropwhile
|
|
from pathlib import Path
|
|
from re import compile as re
|
|
from typing import Tuple
|
|
|
|
from .utils import log
|
|
|
|
__all__ = (
|
|
'Parser',
|
|
'Definition',
|
|
'Module',
|
|
'Package',
|
|
'Function',
|
|
'NestedFunction',
|
|
'Method',
|
|
'Class',
|
|
'NestedClass',
|
|
'AllError',
|
|
'StringIO',
|
|
'ParseError',
|
|
)
|
|
|
|
|
|
class ParseError(Exception):
|
|
"""An error parsing contents of a Python file."""
|
|
|
|
def __str__(self):
|
|
return "Cannot parse file."
|
|
|
|
|
|
class UnexpectedTokenError(ParseError):
|
|
def __init__(self, token, expected_kind):
|
|
self.token = token
|
|
self.expected_kind = expected_kind
|
|
|
|
def __str__(self):
|
|
return "Unexpected token {}, expected {}".format(
|
|
self.token, self.expected_kind
|
|
)
|
|
|
|
|
|
def humanize(string):
|
|
return re(r'(.)([A-Z]+)').sub(r'\1 \2', string).lower()
|
|
|
|
|
|
class Value:
|
|
"""A generic object with a list of preset fields."""
|
|
|
|
def __init__(self, *args):
|
|
if len(self._fields) != len(args):
|
|
raise ValueError(
|
|
'got {} arguments for {} fields for {}: {}'.format(
|
|
len(args),
|
|
len(self._fields),
|
|
self.__class__.__name__,
|
|
self._fields,
|
|
)
|
|
)
|
|
vars(self).update(zip(self._fields, args))
|
|
|
|
def __hash__(self):
|
|
return hash(repr(self))
|
|
|
|
def __eq__(self, other):
|
|
return other and vars(self) == vars(other)
|
|
|
|
def __repr__(self):
|
|
kwargs = ', '.join(
|
|
'{}={!r}'.format(field, getattr(self, field))
|
|
for field in self._fields
|
|
)
|
|
return f'{self.__class__.__name__}({kwargs})'
|
|
|
|
|
|
class Definition(Value):
|
|
"""A Python source code definition (could be class, function, etc)."""
|
|
|
|
_fields = (
|
|
'name',
|
|
'_source',
|
|
'start',
|
|
'end',
|
|
'decorators',
|
|
'docstring',
|
|
'children',
|
|
'callable_args',
|
|
'parent',
|
|
'skipped_error_codes',
|
|
) # type: Tuple[str, ...]
|
|
|
|
_human = property(lambda self: humanize(type(self).__name__))
|
|
kind = property(lambda self: self._human.split()[-1])
|
|
module = property(lambda self: self.parent.module)
|
|
dunder_all = property(lambda self: self.module.dunder_all)
|
|
_slice = property(lambda self: slice(self.start - 1, self.end))
|
|
is_class = False
|
|
|
|
def __iter__(self):
|
|
return chain([self], *self.children)
|
|
|
|
@property
|
|
def error_lineno(self):
|
|
"""Get the line number with which to report violations."""
|
|
if isinstance(self.docstring, Docstring):
|
|
return self.docstring.start
|
|
return self.start
|
|
|
|
@property
|
|
def _publicity(self):
|
|
return {True: 'public', False: 'private'}[self.is_public]
|
|
|
|
@property
|
|
def source(self):
|
|
"""Return the source code for the definition."""
|
|
full_src = self._source[self._slice]
|
|
|
|
def is_empty_or_comment(line):
|
|
return line.strip() == '' or line.strip().startswith('#')
|
|
|
|
filtered_src = dropwhile(is_empty_or_comment, reversed(full_src))
|
|
return ''.join(reversed(list(filtered_src)))
|
|
|
|
def __str__(self):
|
|
out = f'in {self._publicity} {self._human} `{self.name}`'
|
|
if self.skipped_error_codes:
|
|
out += f' (skipping {self.skipped_error_codes})'
|
|
return out
|
|
|
|
|
|
class Module(Definition):
|
|
"""A Python source code module."""
|
|
|
|
_fields = (
|
|
'name',
|
|
'_source',
|
|
'start',
|
|
'end',
|
|
'decorators',
|
|
'docstring',
|
|
'children',
|
|
'parent',
|
|
'_dunder_all',
|
|
'dunder_all_error',
|
|
'future_imports',
|
|
'skipped_error_codes',
|
|
)
|
|
_nest = staticmethod(lambda s: {'def': Function, 'class': Class}[s])
|
|
module = property(lambda self: self)
|
|
dunder_all = property(lambda self: self._dunder_all)
|
|
|
|
@property
|
|
def is_public(self):
|
|
"""Return True iff the module is considered public.
|
|
|
|
This helps determine if it requires a docstring.
|
|
"""
|
|
module_name = Path(self.name).stem
|
|
return not self._is_inside_private_package() and self._is_public_name(
|
|
module_name
|
|
)
|
|
|
|
def _is_inside_private_package(self):
|
|
"""Return True if the module is inside a private package."""
|
|
path = Path(self.name).parent # Ignore the actual module's name.
|
|
syspath = [Path(p) for p in sys.path] # Convert to pathlib.Path.
|
|
|
|
# Bail if we are at the root directory or in `PYTHONPATH`.
|
|
while path != path.parent and path not in syspath:
|
|
if self._is_private_name(path.name):
|
|
return True
|
|
path = path.parent
|
|
|
|
return False
|
|
|
|
def _is_public_name(self, module_name):
|
|
"""Determine whether a "module name" (i.e. module or package name) is public."""
|
|
return not module_name.startswith('_') or (
|
|
module_name.startswith('__') and module_name.endswith('__')
|
|
)
|
|
|
|
def _is_private_name(self, module_name):
|
|
"""Determine whether a "module name" (i.e. module or package name) is private."""
|
|
return not self._is_public_name(module_name)
|
|
|
|
def __str__(self):
|
|
return 'at module level'
|
|
|
|
|
|
class Package(Module):
|
|
"""A package is a __init__.py module."""
|
|
|
|
|
|
class Function(Definition):
|
|
"""A Python source code function."""
|
|
|
|
_nest = staticmethod(
|
|
lambda s: {'def': NestedFunction, 'class': NestedClass}[s]
|
|
)
|
|
|
|
@property
|
|
def is_public(self):
|
|
"""Return True iff this function should be considered public."""
|
|
if self.dunder_all is not None:
|
|
return self.name in self.dunder_all
|
|
else:
|
|
return not self.name.startswith('_')
|
|
|
|
@property
|
|
def is_overload(self):
|
|
"""Return True iff the method decorated with overload."""
|
|
return any(
|
|
decorator.name == "overload" for decorator in self.decorators
|
|
)
|
|
|
|
def is_property(self, property_decorator_names):
|
|
"""Return True if the method is decorated with any property decorator."""
|
|
return any(
|
|
decorator.name in property_decorator_names
|
|
for decorator in self.decorators
|
|
)
|
|
|
|
@property
|
|
def is_test(self):
|
|
"""Return True if this function is a test function/method.
|
|
|
|
We exclude tests from the imperative mood check, because to phrase
|
|
their docstring in the imperative mood, they would have to start with
|
|
a highly redundant "Test that ...".
|
|
|
|
"""
|
|
return self.name.startswith('test') or self.name == 'runTest'
|
|
|
|
@property
|
|
def param_names(self):
|
|
"""Return the parameter names."""
|
|
return self.callable_args
|
|
|
|
|
|
class NestedFunction(Function):
|
|
"""A Python source code nested function."""
|
|
|
|
is_public = False
|
|
|
|
|
|
class Method(Function):
|
|
"""A Python source code method."""
|
|
|
|
@property
|
|
def is_magic(self):
|
|
"""Return True iff this method is a magic method (e.g., `__str__`)."""
|
|
return (
|
|
self.name.startswith('__')
|
|
and self.name.endswith('__')
|
|
and self.name not in VARIADIC_MAGIC_METHODS
|
|
)
|
|
|
|
@property
|
|
def is_init(self):
|
|
"""Return True iff this method is `__init__`."""
|
|
return self.name == '__init__'
|
|
|
|
@property
|
|
def is_public(self):
|
|
"""Return True iff this method should be considered public."""
|
|
# Check if we are a setter/deleter method, and mark as private if so.
|
|
for decorator in self.decorators:
|
|
# Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo'
|
|
if re(fr"^{self.name}\.").match(decorator.name):
|
|
return False
|
|
name_is_public = (
|
|
not self.name.startswith('_')
|
|
or self.name in VARIADIC_MAGIC_METHODS
|
|
or self.is_magic
|
|
)
|
|
return self.parent.is_public and name_is_public
|
|
|
|
@property
|
|
def is_static(self):
|
|
"""Return True iff the method is static."""
|
|
for decorator in self.decorators:
|
|
if decorator.name == "staticmethod":
|
|
return True
|
|
return False
|
|
|
|
|
|
class Class(Definition):
|
|
"""A Python source code class."""
|
|
|
|
_nest = staticmethod(lambda s: {'def': Method, 'class': NestedClass}[s])
|
|
is_public = Function.is_public
|
|
is_class = True
|
|
|
|
|
|
class NestedClass(Class):
|
|
"""A Python source code nested class."""
|
|
|
|
@property
|
|
def is_public(self):
|
|
"""Return True iff this class should be considered public."""
|
|
return (
|
|
not self.name.startswith('_')
|
|
and self.parent.is_class
|
|
and self.parent.is_public
|
|
)
|
|
|
|
|
|
class Decorator(Value):
|
|
"""A decorator for function, method or class."""
|
|
|
|
_fields = 'name arguments'.split()
|
|
|
|
|
|
class Docstring(str):
|
|
"""Represent a docstring.
|
|
|
|
This is a string, but has additional start/end attributes representing
|
|
the start and end of the token.
|
|
|
|
"""
|
|
|
|
def __new__(cls, v, start, end):
|
|
return str.__new__(cls, v)
|
|
|
|
def __init__(self, v, start, end):
|
|
self.start = start
|
|
self.end = end
|
|
|
|
|
|
VARIADIC_MAGIC_METHODS = ('__init__', '__call__', '__new__')
|
|
|
|
|
|
class AllError(Exception):
|
|
"""Raised when there is a problem with __all__ when parsing."""
|
|
|
|
def __init__(self, message):
|
|
"""Initialize the error with a more specific message."""
|
|
Exception.__init__(
|
|
self,
|
|
message
|
|
+ textwrap.dedent(
|
|
"""
|
|
That means pydocstyle cannot decide which definitions are
|
|
public. Variable __all__ should be present at most once in
|
|
each file, in form
|
|
`__all__ = ('a_public_function', 'APublicClass', ...)`.
|
|
More info on __all__: http://stackoverflow.com/q/44834/. ')
|
|
"""
|
|
),
|
|
)
|
|
|
|
|
|
class TokenStream:
|
|
# A logical newline is where a new expression or statement begins. When
|
|
# there is a physical new line, but not a logical one, for example:
|
|
# (x +
|
|
# y)
|
|
# The token will be tk.NL, not tk.NEWLINE.
|
|
LOGICAL_NEWLINES = {tk.NEWLINE, tk.INDENT, tk.DEDENT}
|
|
|
|
def __init__(self, filelike):
|
|
self._generator = tk.generate_tokens(filelike.readline)
|
|
self.current = Token(*next(self._generator, None))
|
|
self.line = self.current.start[0]
|
|
self.log = log
|
|
self.got_logical_newline = True
|
|
|
|
def move(self):
|
|
previous = self.current
|
|
current = self._next_from_generator()
|
|
self.current = None if current is None else Token(*current)
|
|
self.line = self.current.start[0] if self.current else self.line
|
|
is_logical_blank = previous.kind in (tk.NL, tk.COMMENT)
|
|
self.got_logical_newline = (
|
|
previous.kind in self.LOGICAL_NEWLINES
|
|
# Retain logical_newline status if last line was logically blank
|
|
or (self.got_logical_newline and is_logical_blank)
|
|
)
|
|
return previous
|
|
|
|
def _next_from_generator(self):
|
|
try:
|
|
return next(self._generator, None)
|
|
except (SyntaxError, tk.TokenError):
|
|
self.log.warning('error generating tokens', exc_info=True)
|
|
return None
|
|
|
|
def __iter__(self):
|
|
while True:
|
|
if self.current is not None:
|
|
yield self.current
|
|
else:
|
|
return
|
|
self.move()
|
|
|
|
|
|
class TokenKind(int):
|
|
def __repr__(self):
|
|
return "tk.{}".format(tk.tok_name[self])
|
|
|
|
|
|
class Token(Value):
|
|
_fields = 'kind value start end source'.split()
|
|
|
|
def __init__(self, *args):
|
|
super().__init__(*args)
|
|
self.kind = TokenKind(self.kind)
|
|
|
|
def __str__(self):
|
|
return f"{self.kind!r} ({self.value})"
|
|
|
|
|
|
class Parser:
|
|
"""A Python source code parser."""
|
|
|
|
def parse(self, filelike, filename):
|
|
"""Parse the given file-like object and return its Module object."""
|
|
self.log = log
|
|
self.source = filelike.readlines()
|
|
src = ''.join(self.source)
|
|
try:
|
|
compile(src, filename, 'exec')
|
|
except SyntaxError as error:
|
|
raise ParseError() from error
|
|
self.stream = TokenStream(StringIO(src))
|
|
self.filename = filename
|
|
self.dunder_all = None
|
|
self.dunder_all_error = None
|
|
self.future_imports = set()
|
|
self._accumulated_decorators = []
|
|
return self.parse_module()
|
|
|
|
# TODO: remove
|
|
def __call__(self, *args, **kwargs):
|
|
"""Call the parse method."""
|
|
return self.parse(*args, **kwargs)
|
|
|
|
current = property(lambda self: self.stream.current)
|
|
line = property(lambda self: self.stream.line)
|
|
|
|
def consume(self, kind):
|
|
"""Consume one token and verify it is of the expected kind."""
|
|
next_token = self.stream.move()
|
|
if next_token.kind != kind:
|
|
raise UnexpectedTokenError(token=next_token, expected_kind=kind)
|
|
|
|
def leapfrog(self, kind, value=None):
|
|
"""Skip tokens in the stream until a certain token kind is reached.
|
|
|
|
If `value` is specified, tokens whose values are different will also
|
|
be skipped.
|
|
"""
|
|
while self.current is not None:
|
|
if self.current.kind == kind and (
|
|
value is None or self.current.value == value
|
|
):
|
|
self.consume(kind)
|
|
return
|
|
self.stream.move()
|
|
|
|
def parse_docstring(self):
|
|
"""Parse a single docstring and return its value."""
|
|
self.log.debug("parsing docstring, token is %s", self.current)
|
|
while self.current.kind in (tk.COMMENT, tk.NEWLINE, tk.NL):
|
|
self.stream.move()
|
|
self.log.debug(
|
|
"parsing docstring, token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
if self.current.kind == tk.STRING:
|
|
docstring = Docstring(
|
|
self.current.value, self.current.start[0], self.current.end[0]
|
|
)
|
|
self.stream.move()
|
|
return docstring
|
|
return None
|
|
|
|
def parse_decorators(self):
|
|
"""Parse decorators into self._accumulated_decorators.
|
|
|
|
Called after first @ is found.
|
|
Continue to do so until encountering the 'def' or 'class' start token.
|
|
"""
|
|
name = []
|
|
arguments = []
|
|
at_arguments = False
|
|
|
|
while self.current is not None:
|
|
self.log.debug(
|
|
"parsing decorators, current token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
if self.current.kind == tk.NAME and self.current.value in [
|
|
'async',
|
|
'def',
|
|
'class',
|
|
]:
|
|
# Done with decorators - found function or class proper
|
|
break
|
|
elif self.current.kind == tk.OP and self.current.value == '@':
|
|
# New decorator found. Store the decorator accumulated so far:
|
|
self._accumulated_decorators.append(
|
|
Decorator(''.join(name), ''.join(arguments))
|
|
)
|
|
# Now reset to begin accumulating the new decorator:
|
|
name = []
|
|
arguments = []
|
|
at_arguments = False
|
|
elif self.current.kind == tk.OP and self.current.value == '(':
|
|
at_arguments = True
|
|
elif self.current.kind == tk.OP and self.current.value == ')':
|
|
# Ignore close parenthesis
|
|
pass
|
|
elif self.current.kind == tk.NEWLINE or self.current.kind == tk.NL:
|
|
# Ignore newlines
|
|
pass
|
|
else:
|
|
# Keep accumulating current decorator's name or argument.
|
|
if not at_arguments:
|
|
name.append(self.current.value)
|
|
else:
|
|
arguments.append(self.current.value)
|
|
self.stream.move()
|
|
|
|
# Add decorator accumulated so far
|
|
self._accumulated_decorators.append(
|
|
Decorator(''.join(name), ''.join(arguments))
|
|
)
|
|
|
|
def parse_definitions(self, class_, dunder_all=False):
|
|
"""Parse multiple definitions and yield them."""
|
|
while self.current is not None:
|
|
self.log.debug(
|
|
"parsing definition list, current token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
self.log.debug('got_newline: %s', self.stream.got_logical_newline)
|
|
if dunder_all and self.current.value == '__all__':
|
|
self.parse_dunder_all()
|
|
elif (
|
|
self.current.kind == tk.OP
|
|
and self.current.value == '@'
|
|
and self.stream.got_logical_newline
|
|
):
|
|
self.consume(tk.OP)
|
|
self.parse_decorators()
|
|
elif self.current.value in ['def', 'class']:
|
|
yield self.parse_definition(class_._nest(self.current.value))
|
|
elif self.current.kind == tk.INDENT:
|
|
self.consume(tk.INDENT)
|
|
yield from self.parse_definitions(class_)
|
|
elif self.current.kind == tk.DEDENT:
|
|
self.consume(tk.DEDENT)
|
|
return
|
|
elif self.current.value == 'from':
|
|
self.parse_from_import_statement()
|
|
else:
|
|
self.stream.move()
|
|
|
|
def parse_dunder_all(self):
|
|
"""Parse the __all__ definition in a module."""
|
|
assert self.current.value == '__all__'
|
|
self.consume(tk.NAME)
|
|
# More than one __all__ definition means we ignore all __all__.
|
|
if self.dunder_all is not None or self.dunder_all_error is not None:
|
|
self.dunder_all = None
|
|
self.dunder_all_error = 'Could not evaluate contents of __all__. '
|
|
return
|
|
if self.current.value != '=':
|
|
self.dunder_all_error = 'Could not evaluate contents of __all__. '
|
|
return
|
|
self.consume(tk.OP)
|
|
|
|
is_surrounded = False
|
|
if self.current.value in '([':
|
|
is_surrounded = True
|
|
self.consume(tk.OP)
|
|
|
|
dunder_all_content = "("
|
|
while True:
|
|
if is_surrounded and self.current.value in ")]":
|
|
break
|
|
if self.current.kind in (tk.NEWLINE, tk.ENDMARKER):
|
|
break
|
|
if self.current.kind in (tk.NL, tk.COMMENT):
|
|
pass
|
|
elif self.current.kind == tk.STRING or self.current.value == ',':
|
|
dunder_all_content += self.current.value
|
|
else:
|
|
self.dunder_all_error = (
|
|
'Could not evaluate contents of __all__.'
|
|
)
|
|
return
|
|
self.stream.move()
|
|
if is_surrounded:
|
|
self.consume(tk.OP)
|
|
if not is_surrounded and ',' not in dunder_all_content:
|
|
self.dunder_all_error = (
|
|
'Unexpected token kind in __all__: {!r}. '.format(
|
|
self.current.kind
|
|
)
|
|
)
|
|
return
|
|
dunder_all_content += ")"
|
|
|
|
try:
|
|
self.dunder_all = eval(dunder_all_content, {})
|
|
except BaseException as e:
|
|
self.dunder_all_error = (
|
|
'Could not evaluate contents of __all__.'
|
|
'\bThe value was {}. The exception was:\n{}'.format(
|
|
dunder_all_content, e
|
|
)
|
|
)
|
|
|
|
while (
|
|
self.current.kind not in self.stream.LOGICAL_NEWLINES
|
|
and self.current.kind != tk.ENDMARKER
|
|
):
|
|
if self.current.kind != tk.COMMENT:
|
|
self.dunder_all = None
|
|
self.dunder_all_error = (
|
|
'Could not evaluate contents of __all__. '
|
|
)
|
|
return
|
|
self.stream.move()
|
|
|
|
def parse_module(self):
|
|
"""Parse a module (and its children) and return a Module object."""
|
|
self.log.debug("parsing module.")
|
|
start = self.line
|
|
skipped_error_codes = self.parse_skip_comment()
|
|
docstring = self.parse_docstring()
|
|
children = list(self.parse_definitions(Module, dunder_all=True))
|
|
assert self.current is None, self.current
|
|
end = self.line
|
|
cls = Module
|
|
if self.filename.endswith('__init__.py'):
|
|
cls = Package
|
|
module = cls(
|
|
self.filename,
|
|
self.source,
|
|
start,
|
|
end,
|
|
[],
|
|
docstring,
|
|
children,
|
|
None,
|
|
self.dunder_all,
|
|
self.dunder_all_error,
|
|
None,
|
|
skipped_error_codes,
|
|
)
|
|
for child in module.children:
|
|
child.parent = module
|
|
module.future_imports = self.future_imports
|
|
self.log.debug("finished parsing module.")
|
|
return module
|
|
|
|
def parse_definition(self, class_):
|
|
"""Parse a definition and return its value in a `class_` object."""
|
|
start = self.line
|
|
self.consume(tk.NAME)
|
|
name = self.current.value
|
|
self.log.debug("parsing %s '%s'", class_.__name__, name)
|
|
self.stream.move()
|
|
callable_args = []
|
|
if self.current.kind == tk.OP and self.current.value == '(':
|
|
parenthesis_level = 0
|
|
in_default_arg = False
|
|
while True:
|
|
if self.current.kind == tk.OP:
|
|
if self.current.value == '(':
|
|
parenthesis_level += 1
|
|
elif self.current.value == ')':
|
|
parenthesis_level -= 1
|
|
if parenthesis_level == 0:
|
|
break
|
|
elif self.current.value == ',':
|
|
in_default_arg = False
|
|
elif (
|
|
parenthesis_level == 1
|
|
and self.current.kind == tk.NAME
|
|
and not in_default_arg
|
|
):
|
|
callable_args.append(self.current.value)
|
|
in_default_arg = True
|
|
self.stream.move()
|
|
if self.current.kind != tk.OP or self.current.value != ':':
|
|
self.leapfrog(tk.OP, value=":")
|
|
else:
|
|
self.consume(tk.OP)
|
|
if self.current.kind in (tk.NEWLINE, tk.COMMENT):
|
|
skipped_error_codes = self.parse_skip_comment()
|
|
self.leapfrog(tk.INDENT)
|
|
assert self.current.kind != tk.INDENT
|
|
docstring = self.parse_docstring()
|
|
decorators = self._accumulated_decorators
|
|
self.log.debug("current accumulated decorators: %s", decorators)
|
|
self._accumulated_decorators = []
|
|
self.log.debug("parsing nested definitions.")
|
|
children = list(self.parse_definitions(class_))
|
|
self.log.debug(
|
|
"finished parsing nested definitions for '%s'", name
|
|
)
|
|
end = self.line - 1
|
|
else: # one-liner definition
|
|
skipped_error_codes = ''
|
|
docstring = self.parse_docstring()
|
|
decorators = self._accumulated_decorators
|
|
self.log.debug("current accumulated decorators: %s", decorators)
|
|
self._accumulated_decorators = []
|
|
children = []
|
|
end = self.line
|
|
self.leapfrog(tk.NEWLINE)
|
|
definition = class_(
|
|
name,
|
|
self.source,
|
|
start,
|
|
end,
|
|
decorators,
|
|
docstring,
|
|
children,
|
|
callable_args,
|
|
None, # parent
|
|
skipped_error_codes,
|
|
)
|
|
for child in definition.children:
|
|
child.parent = definition
|
|
self.log.debug(
|
|
"finished parsing %s '%s'. Next token is %r",
|
|
class_.__name__,
|
|
name,
|
|
self.current,
|
|
)
|
|
return definition
|
|
|
|
def parse_skip_comment(self):
|
|
"""Parse a definition comment for noqa skips."""
|
|
skipped_error_codes = ''
|
|
while self.current.kind in (tk.COMMENT, tk.NEWLINE, tk.NL):
|
|
if self.current.kind == tk.COMMENT:
|
|
if 'noqa: ' in self.current.value:
|
|
skipped_error_codes = ''.join(
|
|
self.current.value.split('noqa: ')[1:]
|
|
)
|
|
elif self.current.value.startswith('# noqa'):
|
|
skipped_error_codes = 'all'
|
|
self.stream.move()
|
|
self.log.debug(
|
|
"parsing comments before docstring, token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
|
|
if skipped_error_codes:
|
|
break
|
|
|
|
return skipped_error_codes
|
|
|
|
def check_current(self, kind=None, value=None):
|
|
"""Verify the current token is of type `kind` and equals `value`."""
|
|
msg = textwrap.dedent(
|
|
"""
|
|
Unexpected token at line {self.line}:
|
|
|
|
In file: {self.filename}
|
|
|
|
Got kind {self.current.kind!r}
|
|
Got value {self.current.value}
|
|
""".format(
|
|
self=self
|
|
)
|
|
)
|
|
kind_valid = self.current.kind == kind if kind else True
|
|
value_valid = self.current.value == value if value else True
|
|
assert kind_valid and value_valid, msg
|
|
|
|
def parse_from_import_statement(self):
|
|
"""Parse a 'from x import y' statement.
|
|
|
|
The purpose is to find __future__ statements.
|
|
|
|
"""
|
|
self.log.debug('parsing from/import statement.')
|
|
is_future_import = self._parse_from_import_source()
|
|
self._parse_from_import_names(is_future_import)
|
|
|
|
def _parse_from_import_source(self):
|
|
"""Parse the 'from x import' part in a 'from x import y' statement.
|
|
|
|
Return true iff `x` is __future__.
|
|
"""
|
|
assert self.current.value == 'from', self.current.value
|
|
self.stream.move()
|
|
is_future_import = self.current.value == '__future__'
|
|
self.stream.move()
|
|
while (
|
|
self.current is not None
|
|
and self.current.kind in (tk.DOT, tk.NAME, tk.OP)
|
|
and self.current.value != 'import'
|
|
):
|
|
self.stream.move()
|
|
if self.current is None or self.current.value != 'import':
|
|
return False
|
|
self.check_current(value='import')
|
|
assert self.current.value == 'import', self.current.value
|
|
self.stream.move()
|
|
return is_future_import
|
|
|
|
def _parse_from_import_names(self, is_future_import):
|
|
"""Parse the 'y' part in a 'from x import y' statement."""
|
|
if self.current.value == '(':
|
|
self.consume(tk.OP)
|
|
expected_end_kinds = (tk.OP,)
|
|
else:
|
|
expected_end_kinds = (tk.NEWLINE, tk.ENDMARKER)
|
|
while self.current.kind not in expected_end_kinds and not (
|
|
self.current.kind == tk.OP and self.current.value == ';'
|
|
):
|
|
if self.current.kind != tk.NAME:
|
|
self.stream.move()
|
|
continue
|
|
self.log.debug(
|
|
"parsing import, token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
if is_future_import:
|
|
self.log.debug('found future import: %s', self.current.value)
|
|
self.future_imports.add(self.current.value)
|
|
self.consume(tk.NAME)
|
|
self.log.debug(
|
|
"parsing import, token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|
|
if self.current.kind == tk.NAME and self.current.value == 'as':
|
|
self.consume(tk.NAME) # as
|
|
if self.current.kind == tk.NAME:
|
|
self.consume(tk.NAME) # new name, irrelevant
|
|
if self.current.value == ',':
|
|
self.consume(tk.OP)
|
|
self.log.debug(
|
|
"parsing import, token is %r (%s)",
|
|
self.current.kind,
|
|
self.current.value,
|
|
)
|