#!/usr/bin/env python3
"""
|jedi| is mostly being tested by what I would call "integration tests". These
tests are testing type inference with the public API. This makes a
lot of sense for |jedi|. Also, it is hard to write doctests/unittests for
the internal data structures.
There are different kinds of tests:
- completions / inference ``#?``
- goto: ``#!``
- references: ``#<``
How to run tests?
+++++++++++++++++
Jedi uses pytest_ to run unit and integration tests. To run tests,
simply run ``pytest``.
.. _pytest: http://pytest.org
Most integration test cases are located in the ``test/completion`` directory
and each test case starts with one of these comments:
- ``#?`` (completions / inference)
- ``#!`` (goto)
- ``#<`` (references)
There is also support for third party libraries. In a normal test run they are
not being executed, you have to provide a ``--thirdparty`` option.
In addition to pytest's ``-k`` and ``-m`` options, you can use the
``-T`` (``--test-files`) option to specify which test cases should run.
It takes the format of ``FILE_NAME[:LINE[,LINE[,...]]]`` where
``FILE_NAME`` is a file in ``test/completion`` and ``LINE`` is a line
number of the test comment. Here are some examples:
Run tests only in ``completion/basic.py`` and ``completion/imports.py``::
pytest test/test_integration.py -T basic.py -T imports.py
Run test at line 4, 6, and 8 in ``completion/basic.py``::
pytest test/test_integration.py -T basic.py:4,6,8
See ``pytest --help`` for more information.
If you want to debug a test, just use the ``--pdb`` option.
Alternate Test Runner
+++++++++++++++++++++
If you don't like the output of ``pytest``, there's an alternate test runner
that you can start by running ``./run.py``. The above example could be run by::
./run.py basic 4 6 8 50-80
The advantage of this runner is simplicity and more customized error reports.
Auto-Completion Tests
+++++++++++++++++++++
Uses a comment to specify a test on the next line. The comment defines the
expected completions. The comment always begins with `#?`. The last row
symbolizes the cursor. For example::
#? ['upper']
a = 'foo'; a.upp
Inference Tests
+++++++++++++++
Inference tests look very simliar. The difference is that inference tests don't
use brackets::
#? int()
ab = 3; ab
Goto Tests
++++++++++
Goto Tests look like this::
abc = 1
#! ['abc=1']
abc
Additionally it is possible to specify the column by adding a number, which
describes the position of the test (otherwise it's just the end of line)::
#! 2 ['abc=1']
abc
Reference Tests
+++++++++++++++
Tests look like this::
abc = 1
#< (1,0), (3,0)
abc
"""
import os
import re
import sys
import operator
if sys.version_info < (3, 8):
literal_eval = eval
else:
from ast import literal_eval
from io import StringIO
from functools import reduce
from unittest.mock import ANY
from pathlib import Path
import parso
from _pytest.outcomes import Skipped
import pytest
import jedi
from jedi import debug
from jedi.api.classes import Name
from jedi.api.completion import get_user_context
from jedi import parser_utils
from jedi.api.environment import get_default_environment, get_system_environment
from jedi.inference.gradual.conversion import convert_values
from jedi.inference.analysis import Warning
test_dir = Path(__file__).absolute().parent
TEST_COMPLETIONS = 0
TEST_INFERENCE = 1
TEST_GOTO = 2
TEST_REFERENCES = 3
grammar36 = parso.load_grammar(version='3.6')
class BaseTestCase(object):
def __init__(self, skip_version_info=None):
self._skip_version_info = skip_version_info
self._skip = None
def set_skip(self, reason):
self._skip = reason
def get_skip_reason(self, environment):
if self._skip is not None:
return self._skip
if self._skip_version_info is None:
return
comp_map = {
'==': 'eq',
'<=': 'le',
'>=': 'ge',
'<': 'lt',
'>': 'gt',
}
min_version, operator_ = self._skip_version_info
operation = getattr(operator, comp_map[operator_])
if not operation(environment.version_info[:2], min_version):
return "Python version %s %s.%s" % (
operator_, min_version[0], min_version[1]
)
class IntegrationTestCase(BaseTestCase):
def __init__(self, test_type, correct, line_nr, column, start, line,
path=None, skip_version_info=None):
super().__init__(skip_version_info)
self.test_type = test_type
self.correct = correct
self.line_nr = line_nr
self.column = column
self.start = start
self.line = line
self.path = path
self._project = jedi.Project(test_dir)
@property
def module_name(self):
return os.path.splitext(os.path.basename(self.path))[0]
@property
def line_nr_test(self):
"""The test is always defined on the line before."""
return self.line_nr - 1
def __repr__(self):
return '<%s: %s:%s %r>' % (self.__class__.__name__, self.path,
self.line_nr_test, self.line.rstrip())
def script(self, environment):
return jedi.Script(
self.source,
path=self.path,
environment=environment,
project=self._project
)
def run(self, compare_cb, environment=None):
testers = {
TEST_COMPLETIONS: self.run_completion,
TEST_INFERENCE: self.run_inference,
TEST_GOTO: self.run_goto,
TEST_REFERENCES: self.run_get_references,
}
if (self.path.endswith('pytest.py') or self.path.endswith('conftest.py')) \
and environment.executable != os.path.realpath(sys.executable):
# It's not guarantueed that pytest is installed in test
# environments, if we're not running in the same environment that
# we're already in, so just skip that case.
pytest.skip()
return testers[self.test_type](compare_cb, environment)
def run_completion(self, compare_cb, environment):
completions = self.script(environment).complete(self.line_nr, self.column)
# import cProfile; cProfile.run('...')
comp_str = {c.name for c in completions}
for r in completions:
# Test if this access raises an error
assert isinstance(r.type, str)
return compare_cb(self, comp_str, set(literal_eval(self.correct)))
def run_inference(self, compare_cb, environment):
script = self.script(environment)
inference_state = script._inference_state
def comparison(definition):
suffix = '()' if definition.type == 'instance' else ''
return definition.full_name + suffix
def definition(correct, correct_start, path):
should_be = set()
for match in re.finditer('(?:[^ ]+)', correct):
string = match.group(0)
parser = grammar36.parse(string, start_symbol='eval_input', error_recovery=False)
parser_utils.move(parser.get_root_node(), self.line_nr)
node = parser.get_root_node()
module_context = script._get_module_context()
user_context = get_user_context(module_context, (self.line_nr, 0))
node.parent = user_context.tree_node
results = convert_values(user_context.infer_node(node))
if not results:
raise Exception('Could not resolve %s on line %s'
% (match.string, self.line_nr - 1))
should_be |= set(Name(inference_state, r.name) for r in results)
debug.dbg('Finished getting types', color='YELLOW')
# Because the objects have different ids, `repr`, then compare.
should = set(comparison(r) for r in should_be)
return should
should = definition(self.correct, self.start, script.path)
result = script.infer(self.line_nr, self.column)
is_str = set(comparison(r) for r in result)
for r in result:
# Test if this access raises an error
assert isinstance(r.type, str)
return compare_cb(self, is_str, should)
def run_goto(self, compare_cb, environment):
result = self.script(environment).goto(self.line_nr, self.column)
comp_str = str(sorted(str(r.description) for r in result))
return compare_cb(self, comp_str, self.correct)
def run_get_references(self, compare_cb, environment):
result = self.script(environment).get_references(self.line_nr, self.column)
self.correct = self.correct.strip()
compare = sorted(
(('stub:' if r.is_stub() else '')
+ re.sub(r'^completion\.', '', r.module_name),
r.line,
r.column)
for r in result
)
wanted = []
if not self.correct:
positions = []
else:
positions = literal_eval(self.correct)
for pos_tup in positions:
if type(pos_tup[0]) == str:
# this means that there is a module specified
if pos_tup[1] == ...:
pos_tup = pos_tup[0], ANY, pos_tup[2]
wanted.append(pos_tup)
else:
line = pos_tup[0]
if pos_tup[0] is not None:
line += self.line_nr
wanted.append((self.module_name, line, pos_tup[1]))
return compare_cb(self, compare, sorted(wanted))
class StaticAnalysisCase(BaseTestCase):
"""
Static Analysis cases lie in the static_analysis folder.
The tests also start with `#!`, like the inference tests.
"""
def __init__(self, path):
self._path = path
self.name = os.path.basename(path)
with open(path) as f:
self._source = f.read()
skip_version_info = None
for line in self._source.splitlines():
skip_version_info = skip_python_version(line) or skip_version_info
super().__init__(skip_version_info)
def collect_comparison(self):
cases = []
for line_nr, line in enumerate(self._source.splitlines(), 1):
match = re.match(r'(\s*)#! (\d+ )?(.*)$', line)
if match is not None:
column = int(match.group(2) or 0) + len(match.group(1))
cases.append((line_nr + 1, column, match.group(3)))
return cases
def run(self, compare_cb, environment):
def typ_str(inst):
return 'warning ' if isinstance(inst, Warning) else ''
analysis = jedi.Script(
self._source,
path=self._path,
environment=environment,
)._analysis()
analysis = [(r.line, r.column, typ_str(r) + r.name)
for r in analysis]
compare_cb(self, analysis, self.collect_comparison())
def __repr__(self):
return "<%s: %s>" % (self.__class__.__name__, os.path.basename(self._path))
def skip_python_version(line):
# check for python minimal version number
match = re.match(r" *# *python *([<>]=?|==) *(\d+(?:\.\d+)?)$", line)
if match:
minimal_python_version = tuple(map(int, match.group(2).split(".")))
return minimal_python_version, match.group(1)
return None
def collect_file_tests(path, lines, lines_to_execute):
def makecase(t):
return IntegrationTestCase(t, correct, line_nr, column,
start, line, path=path,
skip_version_info=skip_version_info)
start = None
correct = None
test_type = None
skip_version_info = None
for line_nr, line in enumerate(lines, 1):
if correct is not None:
r = re.match(r'^(\d+)\s*(.*)$', correct)
if r:
column = int(r.group(1))
correct = r.group(2)
start += r.regs[2][0] # second group, start index
else:
column = len(line) - 1 # -1 for the \n
if test_type == '!':
yield makecase(TEST_GOTO)
elif test_type == '<':
yield makecase(TEST_REFERENCES)
elif correct.startswith('['):
yield makecase(TEST_COMPLETIONS)
else:
yield makecase(TEST_INFERENCE)
correct = None
else:
skip_version_info = skip_python_version(line) or skip_version_info
try:
r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]*)', line)
# test_type is ? for completion and ! for goto
test_type = r.group(1)
correct = r.group(2)
# Quick hack to make everything work (not quite a bloody unicorn hack though).
if correct == '':
correct = ' '
start = r.start()
except AttributeError:
correct = None
else:
# Skip the test, if this is not specified test.
for l in lines_to_execute:
if isinstance(l, tuple) and l[0] <= line_nr <= l[1] \
or line_nr == l:
break
else:
if lines_to_execute:
correct = None
def collect_dir_tests(base_dir, test_files, check_thirdparty=False):
for f_name in os.listdir(base_dir):
files_to_execute = [a for a in test_files.items() if f_name.startswith(a[0])]
lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, [])
if f_name.endswith(".py") and (not test_files or files_to_execute):
skip = None
if check_thirdparty:
lib = f_name.replace('_.py', '')
try:
# there is always an underline at the end.
# It looks like: completion/thirdparty/pylab_.py
__import__(lib)
except ImportError:
skip = 'Thirdparty-Library %s not found.' % lib
path = os.path.join(base_dir, f_name)
with open(path, newline='') as f:
source = f.read()
for case in collect_file_tests(path, StringIO(source),
lines_to_execute):
case.source = source
if skip:
case.set_skip(skip)
yield case
docoptstr = """
Using run.py to make debugging easier with integration tests.
An alternative testing format, which is much more hacky, but very nice to
work with.
Usage:
run.py [--pdb] [--debug] [--thirdparty] [--env <dotted>] [<rest>...]
run.py --help
Options:
-h --help Show this screen.
--pdb Enable pdb debugging on fail.
-d, --debug Enable text output debugging (please install ``colorama``).
--thirdparty Also run thirdparty tests (in ``completion/thirdparty``).
--env <dotted> A Python version, like 3.9, 3.8, etc.
"""
if __name__ == '__main__':
import docopt
arguments = docopt.docopt(docoptstr)
import time
t_start = time.time()
if arguments['--debug']:
jedi.set_debug_function()
# get test list, that should be executed
test_files = {}
last = None
for arg in arguments['<rest>']:
match = re.match(r'(\d+)-(\d+)', arg)
if match:
start, end = match.groups()
test_files[last].append((int(start), int(end)))
elif arg.isdigit():
if last is None:
continue
test_files[last].append(int(arg))
else:
test_files[arg] = []
last = arg
# completion tests:
dir_ = os.path.dirname(os.path.realpath(__file__))
completion_test_dir = os.path.join(dir_, '../test/completion')
completion_test_dir = os.path.abspath(completion_test_dir)
tests_fail = 0
# execute tests
cases = list(collect_dir_tests(completion_test_dir, test_files))
if test_files or arguments['--thirdparty']:
completion_test_dir += '/thirdparty'
cases += collect_dir_tests(completion_test_dir, test_files, True)
def file_change(current, tests, fails):
if current is None:
current = ''
else:
current = os.path.basename(current)
print('{:25} {} tests and {} fails.'.format(current, tests, fails))
def report(case, actual, desired):
if actual == desired:
return 0
else:
print("\ttest fail @%d, actual = %s, desired = %s"
% (case.line_nr - 1, actual, desired))
return 1
if arguments['--env']:
environment = get_system_environment(arguments['--env'])
else:
# Will be 3.6.
environment = get_default_environment()
import traceback
current = cases[0].path if cases else None
count = fails = 0
for c in cases:
if c.get_skip_reason(environment):
continue
if current != c.path:
file_change(current, count, fails)
current = c.path
count = fails = 0
try:
if c.run(report, environment):
tests_fail += 1
fails += 1
except Exception:
traceback.print_exc()
print("\ttest fail @%d" % (c.line_nr - 1))
tests_fail += 1
fails += 1
if arguments['--pdb']:
import pdb
pdb.post_mortem()
except Skipped:
pass
count += 1
file_change(current, count, fails)
print('\nSummary: (%s fails of %s tests) in %.3fs'
% (tests_fail, len(cases), time.time() - t_start))
exit_code = 1 if tests_fail else 0
sys.exit(exit_code)