From a18ad3aaadd2667e0cfab01449761bd121079f36 Mon Sep 17 00:00:00 2001 From: Marius Smytzek Date: Wed, 28 Jun 2023 11:49:20 +0200 Subject: [PATCH] added pytest runner --- BugsInPy.ipynb | 109 +++++--- .../subjects/tests/test_runner/middle.py | 13 + .../tests/test_runner/tests/test_middle.py | 13 + src/sflkit/runners/__init__.py | 4 +- src/sflkit/runners/run.py | 238 +++++++++++++++++- tests/test_runner.py | 144 +++++++++++ tests/utils.py | 2 +- 7 files changed, 486 insertions(+), 37 deletions(-) create mode 100644 resources/subjects/tests/test_runner/middle.py create mode 100644 resources/subjects/tests/test_runner/tests/test_middle.py create mode 100644 tests/test_runner.py diff --git a/BugsInPy.ipynb b/BugsInPy.ipynb index 4de8ed6..ece588c 100644 --- a/BugsInPy.ipynb +++ b/BugsInPy.ipynb @@ -26,7 +26,31 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, + "id": "6fe1d278", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting whatthepatch\n", + " Downloading whatthepatch-1.0.5-py3-none-any.whl (11 kB)\n", + "Installing collected packages: whatthepatch\n", + "Successfully installed whatthepatch-1.0.5\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!pip install whatthepatch" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "71944b13-ca1f-4f5b-b1dd-7f0447e01626", "metadata": {}, "outputs": [], @@ -54,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "id": "7a4db981-8306-4bb0-88d8-5b3f711adaee", "metadata": {}, "outputs": [], @@ -65,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 7, "id": "6d269bfe-e9eb-43ac-977f-6bed9080173e", "metadata": {}, "outputs": [], @@ -76,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "id": "41cfb43a-fa07-49d4-82e3-b1a0cb04971f", "metadata": {}, "outputs": [], @@ -90,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 9, "id": "84c154e6-9bd3-4a46-9604-73a895457f45", "metadata": {}, "outputs": [], @@ -104,17 +128,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 10, "id": "54609932-e26f-4fd1-86b6-b6e49fa78fde", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'/home/marius/sflkit'" + "'/Users/marius/Desktop/work/projects/sflkit'" ] }, - "execution_count": 6, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -126,17 +150,17 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "id": "93d35204-e3a9-4911-a7cf-a588d08fc2ef", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'/home/marius/sflkit/events'" + "'/Users/marius/Desktop/work/projects/sflkit/events'" ] }, - "execution_count": 7, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -148,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "id": "deb300c5-7fd1-4ada-9198-566cdd859b61", "metadata": {}, "outputs": [], @@ -167,7 +191,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 13, "id": "106b4a86-48a2-46b1-94fe-e447df2b7647", "metadata": {}, "outputs": [], @@ -187,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 14, "id": "cb6dc70f-82cc-4475-9cbe-3d481e914cea", "metadata": {}, "outputs": [], @@ -215,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 15, "id": "65b9d0a4-2d20-4326-88a2-8a7a6a9ee8f2", "metadata": {}, "outputs": [], @@ -227,18 +251,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 16, "id": "37e38835-1276-4db0-9a64-41d3e13df982", "metadata": {}, "outputs": [], "source": [ "excluded_subjects = [matplotlib, pandas, spacy]\n", - "max_bugs_per_subject = 3" + "max_bugs_per_subject = 500" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 17, "id": "ce05a37b-ae11-4009-9e48-d1a5b65ff51a", "metadata": {}, "outputs": [], @@ -249,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 18, "id": "362b469d-6c69-4642-8932-a5ea97873716", "metadata": {}, "outputs": [], @@ -265,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 19, "id": "b67456e2-3039-4a4d-a45c-bc9c40d3583d", "metadata": {}, "outputs": [], @@ -275,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 20, "id": "97fb587c-8706-4436-94f9-7ff3dcd79e4b", "metadata": {}, "outputs": [], @@ -291,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 21, "id": "f832b7ac-1b57-4c42-acf3-c05e2796850e", "metadata": {}, "outputs": [], @@ -302,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 22, "id": "4d2c4307-626c-4c7c-8494-d641fbe5cfa0", "metadata": {}, "outputs": [], @@ -318,7 +342,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 23, "id": "a02a4869-32d7-4e92-bd5a-7651820f8332", "metadata": {}, "outputs": [], @@ -331,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 24, "id": "9dea83dd-812b-43d7-8579-7963a3d6e6cf", "metadata": {}, "outputs": [], @@ -359,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 25, "id": "21c43449-662d-42c9-b8e4-2ad5099948b9", "metadata": {}, "outputs": [], @@ -374,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 26, "id": "8fcfecfb-ddd0-4173-9b87-1da35a89ff87", "metadata": {}, "outputs": [], @@ -386,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 27, "id": "1cf603b1-a683-4d47-8fe9-bb461722548c", "metadata": {}, "outputs": [], @@ -467,7 +491,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 28, "id": "d6e48520-de25-44aa-8e80-6f6ab1a6f086", "metadata": {}, "outputs": [], @@ -503,7 +527,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 29, "id": "d22a6a38-393a-4e91-8f0a-fa513a87d37f", "metadata": {}, "outputs": [], @@ -516,10 +540,29 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 30, "id": "7ddd63f4-5d9b-46e1-b1ad-e6bbe630cb6b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pytest did not generate file\n" + ] + }, + { + "ename": "AssertionError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[30], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[39massert\u001b[39;00m verify_correct_version(test_project, test_bug_id, test_test_file)\n", + "\u001b[0;31mAssertionError\u001b[0m: " + ] + } + ], "source": [ "assert verify_correct_version(test_project, test_bug_id, test_test_file)" ] @@ -2208,7 +2251,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.10" + "version": "3.10.4" } }, "nbformat": 4, diff --git a/resources/subjects/tests/test_runner/middle.py b/resources/subjects/tests/test_runner/middle.py new file mode 100644 index 0000000..849d5cb --- /dev/null +++ b/resources/subjects/tests/test_runner/middle.py @@ -0,0 +1,13 @@ +def middle(x, y, z): + m = z + if y < z: + if x < y: + m = y + elif x < z: + m = y # bug + else: + if x > y: + m = y + elif x > z: + m = x + return m diff --git a/resources/subjects/tests/test_runner/tests/test_middle.py b/resources/subjects/tests/test_runner/tests/test_middle.py new file mode 100644 index 0000000..4d56fe3 --- /dev/null +++ b/resources/subjects/tests/test_runner/tests/test_middle.py @@ -0,0 +1,13 @@ +import unittest +from middle import middle + + +class MiddleTests(unittest.TestCase): + def test_213(self): + self.assertEqual(middle(2, 1, 3), 2) + + def test_321(self): + self.assertEqual(middle(3, 2, 1), 2) + + def test_312(self): + self.assertEqual(middle(3, 1, 2), 2) diff --git a/src/sflkit/runners/__init__.py b/src/sflkit/runners/__init__.py index 709d8bc..2371a2d 100644 --- a/src/sflkit/runners/__init__.py +++ b/src/sflkit/runners/__init__.py @@ -1,6 +1,6 @@ import enum -from sflkit.runners.run import Runner, VoidRunner +from sflkit.runners.run import Runner, VoidRunner, PytestRunner, UnittestRunner class RunnerType(enum.Enum): @@ -8,3 +8,5 @@ def __init__(self, runner: Runner): self.runner = runner VOID_RUNNER = VoidRunner + PYTEST_RUNNER = PytestRunner + UNITTEST_RUNNER = UnittestRunner diff --git a/src/sflkit/runners/run.py b/src/sflkit/runners/run.py index c9aff25..829f4c4 100644 --- a/src/sflkit/runners/run.py +++ b/src/sflkit/runners/run.py @@ -1,6 +1,240 @@ -class Runner: - pass +import abc +import enum +import os +import re +import shutil +import subprocess +from pathlib import Path +from typing import List, Dict, Optional, Any + +Environment = Dict[str, str] + +PYTEST_COLLECT_PATTERN = re.compile( + '<(?PPackage|Module|Class|Function|UnitTestCase|TestCaseFunction) (?P[^>]+|"([^"]|\\")+")>' +) +PYTEST_RESULT_PATTERN = re.compile( + rb"= ((((?P\d+) failed)|((?P

\d+) passed)|(\d+ warnings?))(, )?)+ in " +) + + +class PytestNode(abc.ABC): + def __init__(self, name: str, parent=None): + self.name = name + self.parent: Optional[PytestNode] = parent + self.children: List[PytestNode] = [] + + def __repr__(self): + return self.name + + def __str__(self): + return self.__repr__() + + @abc.abstractmethod + def visit(self) -> List[Any]: + pass + + +class Package(PytestNode): + def visit(self) -> List[Any]: + return sum([node.visit() for node in self.children], start=[]) + + def __repr__(self): + if self.parent: + return os.path.join(repr(self.parent), self.name) + else: + return self.name + + +class Module(PytestNode): + def visit(self) -> List[Any]: + return sum([node.visit() for node in self.children], start=[]) + + def __repr__(self): + if self.parent: + return os.path.join(repr(self.parent), self.name) + else: + return self.name + + +class Class(PytestNode): + def visit(self) -> List[Any]: + return sum([node.visit() for node in self.children], start=[]) + + def __repr__(self): + if self.parent: + return f"{repr(self.parent)}::{self.name}" + else: + return f"::{self.name}" + + +class Function(PytestNode): + def visit(self) -> List[Any]: + return [self] + sum([node.visit() for node in self.children], start=[]) + + def __repr__(self): + if self.parent: + return f"{repr(self.parent)}::{self.name}" + else: + return f"::{self.name}" + + +class PytestTree: + def __init__(self): + self.roots: List[PytestNode] = [] + + @staticmethod + def _count_spaces(s: str): + return len(s) - len(s.lstrip()) + + def parse(self, output: str, directory: Path = None): + current_level = 0 + current_node = None + for line in output.split("\n"): + match = PYTEST_COLLECT_PATTERN.search(line) + if match: + level = self._count_spaces(line) // 2 + name = match.group("name") + if match.group("kind") == "Package": + node_class = Package + if directory: + name = os.path.relpath(name, directory) + elif match.group("kind") == "Module": + node_class = Module + if directory: + name = os.path.relpath(name, directory) + elif match.group("kind") in ("Class", "UnitTestCase"): + node_class = Class + elif match.group("kind") in ("Function", "TestCaseFunction"): + node_class = Function + else: + continue + node = node_class(name) + if level == 0: + current_node = node + current_level = 0 + self.roots.append(node) + elif level > current_level: + current_node.children.append(node) + node.parent = current_node + current_node = node + current_level = level + else: + for _ in range(current_level - level + 1): + if current_node.parent: + current_node = current_node.parent + current_node.children.append(node) + node.parent = current_node + current_node = node + current_level = level + + def visit(self): + return sum([node.visit() for node in self.roots], start=[]) + + +class TestResult(enum.Enum): + PASSING = "PASSING" + FAILING = "FAILING" + UNDEFINED = "UNDEFINED" + + def get_dir(self): + return self.value.lower() + + +class Runner(abc.ABC): + def __init__(self, re_filter: str = r".*"): + self.re_filter = re.compile(re_filter) + + def get_tests(self, directory: Path, environ: Environment = None) -> List[str]: + return [] + + def run_test( + self, directory: Path, test: str, environ: Environment = None + ) -> TestResult: + return TestResult.UNDEFINED + + def filter_tests(self, tests: List[str]) -> List[str]: + return list(filter(self.re_filter.search, tests)) + + def run_tests( + self, + directory: Path, + output: Path, + tests: List[str], + environ: Environment = None, + ): + output.mkdir(parents=True, exist_ok=True) + for test_result in TestResult: + (output / test_result.get_dir()).mkdir(parents=True, exist_ok=True) + for run_id, test in enumerate(tests): + test_result = self.run_test(directory, test, environ=environ) + if os.path.exists(directory / "EVENTS_PATH"): + shutil.move( + directory / "EVENTS_PATH", + output / test_result.get_dir() / str(run_id), + ) + + def run(self, directory: Path, output: Path, environ: Environment = None): + self.run_tests( + directory, + output, + self.filter_tests(self.get_tests(directory, environ=environ)), + environ=environ, + ) class VoidRunner(Runner): pass + + +class PytestRunner(Runner): + def get_tests(self, directory: Path, environ: Environment = None) -> List[str]: + output = subprocess.run( + ["python", "-m", "pytest", "--collect-only"], + stdout=subprocess.PIPE, + env=environ, + cwd=directory, + ).stdout.decode("utf-8") + tree = PytestTree() + tree.parse(output, directory=directory) + return list(map(str, tree.visit())) + + @staticmethod + def __get_pytest_result__( + output: bytes, + ) -> tuple[bool, Optional[int], Optional[int]]: + match = PYTEST_RESULT_PATTERN.search(output) + if match: + if match.group("f"): + failing = int(match.group("f")) + else: + failing = 0 + if match.group("p"): + passing = int(match.group("p")) + else: + passing = 0 + return True, passing, failing + return False, None, None + + def run_test( + self, directory: Path, test: str, environ: Environment = None + ) -> TestResult: + output = subprocess.run( + ["python", "-m", "pytest", test], + stdout=subprocess.PIPE, + env=environ, + cwd=directory, + ).stdout + successful, passing, failing = self.__get_pytest_result__(output) + if successful: + if passing > 0 and failing == 0: + return TestResult.PASSING + elif failing > 0 and passing == 0: + return TestResult.FAILING + else: + return TestResult.UNDEFINED + else: + return TestResult.UNDEFINED + + +class UnittestRunner(Runner): + pass diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..90b3ef8 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,144 @@ +import os +from pathlib import Path + +from sflkit import Config, instrument_config, Analyzer +from sflkit.analysis.analysis_type import AnalysisType +from sflkit.analysis.suggestion import Location +from sflkit.model import EventFile +from sflkit.runners.run import PytestTree, Package, Module, Function, PytestRunner +from tests.utils import BaseTest + + +class RunnerTests(BaseTest): + PYTEST_COLLECT = ( + "================================================================================================" + "============================= test session starts ==============================================" + "================================================================================\n" + "platform darwin -- Python 3.8.4, pytest-7.3.1, pluggy-1.0.0\n" + "rootdir: /Users/test/test\n" + "collected 5 items\n\n" + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n" + "================================================================================================" + "=============================== warnings summary ===============================================" + "================================================================================\n" + "python3.8/site-packages/future/standard_library/__init__.py:65: DeprecationWarning: " + "the imp module is deprecated in favour of importlib; see the module's documentation for " + "alternative uses\n" + " import imp\n\n" + "-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html\n" + "================================================================================================" + "========================== 5 tests collected in 0.05s ==========================================" + "================================================================================\n" + ) + + def test_pytest_tree_parser(self): + tree = PytestTree() + tree.parse(self.PYTEST_COLLECT) + self.assertEqual(1, len(tree.roots)) + package_tests = tree.roots[0] + self.assertIsInstance(package_tests, Package) + self.assertEqual(2, len(package_tests.children)) + self.assertEqual("tests", package_tests.name) + + module_1, package_1 = package_tests.children + self.assertIsInstance(module_1, Module) + self.assertEqual(2, len(module_1.children)) + self.assertEqual("1.py", module_1.name) + + test_1, test_2 = module_1.children + self.assertIsInstance(test_1, Function) + self.assertEqual(0, len(test_1.children)) + self.assertEqual("test_1", test_1.name) + self.assertIsInstance(test_2, Function) + self.assertEqual(0, len(test_2.children)) + self.assertEqual("test_2", test_2.name) + + self.assertIsInstance(package_1, Package) + self.assertEqual(2, len(package_1.children)) + self.assertEqual("1", package_1.name) + + module_2, module_3 = package_1.children + self.assertIsInstance(module_2, Module) + self.assertEqual(2, len(module_2.children)) + self.assertEqual("2.py", module_2.name) + + test_3, test_4 = module_2.children + self.assertIsInstance(test_3, Function) + self.assertEqual(0, len(test_3.children)) + self.assertEqual("test_3", test_3.name) + self.assertIsInstance(test_4, Function) + self.assertEqual(0, len(test_4.children)) + self.assertEqual("test_4", test_4.name) + + self.assertIsInstance(module_3, Module) + self.assertEqual(1, len(module_3.children)) + self.assertEqual("3.py", module_3.name) + + test_5 = module_3.children[0] + self.assertIsInstance(test_5, Function) + self.assertEqual(0, len(test_5.children)) + self.assertEqual("test_5", test_5.name) + + def test_pytest_tree_visit(self): + tree = PytestTree() + tree.parse(self.PYTEST_COLLECT) + tests = list(map(str, tree.visit())) + expected = [ + "tests/1.py::test_1", + "tests/1.py::test_2", + "tests/1/2.py::test_3", + "tests/1/2.py::test_4", + "tests/1/3.py::test_5", + ] + self.assertEqual(expected, tests) + + def assertPathExists(self, path: os.PathLike): + self.assertTrue(os.path.exists(path), f"{path} does not exists.") + + def test_runner(self): + config = Config.create( + path=os.path.join(self.TEST_RESOURCES, BaseTest.TEST_RUNNER), + language="python", + events="line", + predicates="line", + working=BaseTest.TEST_DIR, + exclude="tests", + ) + instrument_config(config) + runner = PytestRunner() + output = Path(BaseTest.TEST_DIR, "events").absolute() + runner.run(Path(BaseTest.TEST_DIR), output) + self.assertPathExists(output) + self.assertPathExists(output / "passing") + self.assertPathExists(output / "failing") + self.assertPathExists(output / "undefined") + analyzer = Analyzer( + [ + EventFile( + output / "failing" / os.listdir(output / "failing")[0], + 0, + failing=True, + ) + ], + [ + EventFile(output / "passing" / path, run_id) + for run_id, path in enumerate(os.listdir(output / "passing"), start=1) + ], + config.factory, + ) + analyzer.analyze() + predicates = analyzer.get_analysis_by_type(AnalysisType.LINE) + suggestions = sorted(map(lambda p: p.get_suggestion(), predicates)) + self.assertEqual(1, suggestions[-1].suspiciousness) + self.assertEqual(1, len(suggestions[-1].lines)) + self.assertEqual(Location("middle.py", 7), suggestions[-1].lines[0]) diff --git a/tests/utils.py b/tests/utils.py index d0683fe..4dcabea 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -10,7 +10,6 @@ class BaseTest(unittest.TestCase): - TEST_RESOURCES = os.path.join("resources", "subjects", "tests") TEST_DIR = "test_dir" TEST_EVENTS = "test_events.json" @@ -24,6 +23,7 @@ class BaseTest(unittest.TestCase): TEST_LOOP = "test_loop" TEST_PROPERTIES = "test_properties" TEST_SPECIAL_VALUES = "test_special_values" + TEST_RUNNER = "test_runner" DELTA = 0.0000001 EVENTS = [