Merge pull request #581 from yorukot/testsuite

Add python testsuite

src/internal/config_function.go

@@ -41,8 +41,7 @@ func initialConfig(dir string) (toggleDotFileBool bool, toggleFooter bool, first
 	slog.SetDefault(slog.New(slog.NewTextHandler(
 		file, &slog.HandlerOptions{Level: logLevel})))
 
-	slog.Debug("Runtime information", "runtime.GOOS", runtime.GOOS)
-
+
 	loadHotkeysFile()
 
 	loadThemeFile()
@@ -93,6 +92,9 @@ func initialConfig(dir string) (toggleDotFileBool bool, toggleFooter bool, first
 		firstFilePanelDir = variable.HomeDir
 	}
 
+	slog.Debug("Runtime information", "runtime.GOOS", runtime.GOOS,
+		"start directory", firstFilePanelDir)
+
 	return toggleDotFileBool, toggleFooter, firstFilePanelDir
 }
 

src/internal/model.go

@@ -462,4 +462,5 @@ func (m *model) quitSuperfile() {
 		currentDir = strings.ReplaceAll(currentDir, "'", "'\\''")
 		os.WriteFile(variable.SuperFileStateDir+"/lastdir", []byte("cd '"+currentDir+"'"), 0755)
 	}
+	slog.Debug("Quitting superfile", "current dir", currentDir)
 }

testsuite/.gitignore

@@ -0,0 +1,9 @@
+# python venv site packages
+site-packages/
+
+#python venvs
+.venv/
+
+# python pycache
+__pycache__/
+*.pyc

testsuite/Notes.md

@@ -0,0 +1,68 @@
+# Implementation notes
+
+- The `pyautogui` sends input to the process in focus, which is the `spf` subprocess.
+- If `spf` is not exited correcly via `q`, it causes wierd vertical tabs in print statements from python
+- There is some flakiness in sending of input. Many times, `Ctrl+C` is received as `C` in `spf`
+  - If first key is `Ctrl+C`, its always received as `C`
+- Note : You must keep your focus on the terminal for the entire duration of test run. `pyautogui` sends keypress to process on focus.
+
+## Input to spf
+
+### Pyautogui alternatives
+POC with pyautogui as a lot of issues, stated above.
+
+#### Linux / MacOS
+
+- xdotool
+  - Seems complicated. It wont be able to manage spf process that well
+- mkfifo / Manual linux piping
+  - Too much manual work to send inputs, even if it works
+- tmux
+  - Supports full terminal programs and has a python wrapper library
+  - See `docs/tmux.md`
+  - Not available for windows
+- References
+  - https://superuser.com/questions/585398/sending-simulated-keystrokes-in-bash
+
+#### Windows
+
+- Autohotkey
+  - No better than pyautogui
+- ControlSend and SendInput utility in windows
+  - Isn't that just for C# / C++ code ?
+- Python ctypes
+  - https://stackoverflow.com/questions/62189991/how-to-wrap-the-sendinput-function-to-python-using-ctypes
+- pywin32 library
+  - Create a new GUI window for test
+  - Use `win32gui.SendMessage` or `win32gui.PostMessage`
+  - Probably the correct way, but I havent been able to get it working.
+  - First we need to get it send input to a sample window like notepad, etc. Then we can make superfile work
+- pywinpty
+  - Heavy installations requirements. Needs Rust, and Visual studio build tools.
+  - Rust cargo not found
+    - Needs rust 
+  - link.exe not found (` the msvc targets depend on the msvc linker but link.exe was not found` )
+    - Needs to install Visual Studio Build Tools (build tools and spectre mitigated libs)
+    - Had to manually find link.exe and put it on the PATH
+  - You might get error of unable to find mspdbcore.dll (I havent been able to solve it so far)
+    - https://stackoverflow.com/questions/67328795/c1356-unable-to-find-mspdbcore-dll
+- References
+  - https://www.reddit.com/r/tmux/comments/l580mi/is_there_a_tmuxlike_equivalent_for_windows/
+
+## Directory setup
+- Programmatic setup is better.
+- We could keep test directory setup as a config file - json/yaml/toml 
+- or as a hardcoded python dict
+- Turns out, a in-memory fs is better. We have utilities like copy to actual fs and print tree
+  - Although it has a limitation of not being able to work with large files, as that would consume a lot of RAM
+  - For large files, we could do actually make them only on the actual filesystem, and not use in-memory fs
+  - https://docs.pyfilesystem.org/en/latest/reference/memoryfs.html
+  - https://docs.pyfilesystem.org/en/latest/guide.html 
+
+
+## Tests and Validation
+- Each tests starts independently, so there is no strict order
+- Hardcoded validations . Predefined test, where each test has start dir, key press, and validations
+- We could have a base Class test. where check(), input(), init(), methods would be overrided
+- It allows greater flexibility in terms of testcases.
+- Abstraction layer for spf init, teardown and inputm

testsuite/ReadMe.md

@@ -0,0 +1,78 @@
+## Coding style rules
+- Prefer using strong typing 
+- Prefer using type hinting for the first time the variable is declared, and for functions paremeters and return types
+- Use `-> None` to explicitly indicate no return value
+
+### Ideas
+- Recommended to integrate your IDE with PEP8 to highlight PEP8 violations in real-time
+- Enforcing PEP8 via `pylint flake8 pycodestyle` and via pre commit hooks
+
+## Writing New testcases
+- Just create a file ending with `_test.py` in `tests` directory
+  - Any subclass of BaseTest with name ending with `Test` will be executed 
+  - see `run_tests` and `get_testcases` in `core/runner.py` for more info
+
+## Setup 
+Requires python 3.9 or later.
+
+## Setup for MacOS / Linux
+
+### Install tmux
+- You need to have tmux installed. See https://github.com/tmux/tmux/wiki
+
+### Python virtual env setup
+```
+# cd to this directory
+cd <path/to/here>
+python3 -m venv .venv
+.venv/bin/pip install --upgrade pip
+.venv/bin/pip install -r requirements.txt
+```
+
+### Make sure you build spf
+```
+# cd to the superfile repo root (parent of this)
+cd <superfile_root>
+./build.sh
+```
+
+### Running testsuite
+```
+.venv/bin/python3 main.py
+```
+## Setup for Windows
+Coming soon.
+
+
+
+### Python virtual env setup
+```
+# cd to this directory
+cd <path/to/here>
+python3 -m venv .venv
+.venv\Scripts\python -m pip install --upgrade pip
+.venv\Scripts\pip -r requirements.txt
+```
+
+### Make sure you build spf
+```
+# cd to the superfile repo root (parent of this)
+cd <superfile_root>
+go build -o bin/spf.exe
+```
+
+### Running testsuite
+Notes 
+- You must keep your focus on the terminal for the entire duration of test run. `pyautogui` sends keypress to process on focus.
+
+```
+.venv\Scripts\python main.py
+```
+
+## Tips while running tests
+- Use `-d` or `--debug` to enable debug logs during test run.
+- If you see flakiness in test runs due to superfile being still open, consider using `--close-wait-time` options to increase wait time for superfile to close
+- Use `-t` or `--tests` to only run specific tests
+  - Example `python main.py -d -t RenameTest CopyTest`
+
+- If you see `libtmux` errors like `libtmux.exc.LibTmuxException: ['no server running on /private/tmp/tmux-501/superfile']` Make sure your python version is up to date

testsuite/core/base_test.py

@@ -0,0 +1,108 @@
+import logging
+import time
+from abc import ABC, abstractmethod
+from core.environment import Environment
+from pathlib import Path
+from typing import Union, List, Tuple
+import core.keys as keys
+import core.test_constants as tconst
+
+
+class BaseTest(ABC):
+    """Base class for all tests
+    The idea is to have independency among each test.
+    And for each test to have full control on its environment, execution, and validation.
+    """
+    def __init__(self, test_env : Environment):
+        self.env = test_env
+        self.logger = logging.getLogger()
+
+    @abstractmethod
+    def setup(self) -> None:
+        """Set up the required things for test
+        """
+
+    @abstractmethod
+    def test_execute(self) -> None:
+        """Execute the test
+        """
+
+    @abstractmethod
+    def validate(self) -> bool:
+        """Validate that test passed. Log exception if failed.
+        Returns:
+            bool: True if validation passed
+        """
+
+class GenericTestImpl(BaseTest):
+    def __init__(self, test_env : Environment,
+        test_root : Path,
+        start_dir : Path,
+        test_dirs : List[Path],
+        test_files : List[Tuple[Path, str]],
+        key_inputs : List[Union[keys.Keys,str]],
+        validate_exists : List[Path] = [],
+        validate_not_exists : List[Path] = []):
+        super().__init__(test_env)
+        self.test_root = test_root
+        self.start_dir = start_dir
+        self.test_dirs = test_dirs
+        self.test_files = test_files
+        self.key_inputs = key_inputs
+        self.validate_exists = validate_exists
+        self.validate_not_exists = validate_not_exists
+
+    def setup(self) -> None:
+        for dir_path in self.test_dirs:
+            self.env.fs_mgr.makedirs(dir_path)
+        for file_path, data in self.test_files:
+            self.env.fs_mgr.create_file(file_path, data)
+
+        self.logger.debug("Current file structure : \n%s",
+            self.env.fs_mgr.tree(self.test_root))
+
+
+    def test_execute(self) -> None:
+        """Execute the test
+        """
+        # Start in DIR1
+        self.env.spf_mgr.start_spf(self.env.fs_mgr.abspath(self.start_dir))
+
+        assert self.env.spf_mgr.is_spf_running(), "Superfile is not running"
+
+        for cur_input in self.key_inputs:
+            if isinstance(cur_input, keys.Keys):
+                self.env.spf_mgr.send_special_input(cur_input)
+            else:
+                assert isinstance(cur_input, str), "Invalid input type"
+                self.env.spf_mgr.send_text_input(cur_input)
+            time.sleep(tconst.KEY_DELAY)
+
+        time.sleep(tconst.OPERATION_DELAY)
+        self.env.spf_mgr.send_special_input(keys.KEY_ESC)    
+        time.sleep(tconst.CLOSE_WAIT_TIME)
+        self.logger.debug("Finished Execution")
+
+    def validate(self) -> bool:
+        """Validate that test passed. Log exception if failed.
+        Returns:
+            bool: True if validation passed
+        """
+        self.logger.debug("spf_manager info : %s, Current file structure : \n%s",
+            self.env.spf_mgr.runtime_info(), self.env.fs_mgr.tree(self.test_root))
+        try:
+            assert not self.env.spf_mgr.is_spf_running(), "Superfile is still running"
+            for file_path in self.validate_exists:
+                assert self.env.fs_mgr.check_exists(file_path), f"File {file_path} does not exists"
+
+            for file_path in self.validate_not_exists:
+                assert not self.env.fs_mgr.check_exists(file_path), f"File {file_path} exists" 
+        except AssertionError as ae:
+            self.logger.debug("Test assertion failed : %s", ae, exc_info=True)
+            return False
+
+        return True
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}"
+

testsuite/core/environment.py

@@ -0,0 +1,14 @@
+from core.spf_manager import BaseSPFManager
+from core.fs_manager import TestFSManager
+
+class Environment:
+    """Manage test environment
+    Manage cleanup of environment and other stuff at a single place
+    """    
+    def __init__(self, spf_manager : BaseSPFManager, fs_manager : TestFSManager ):
+        self.spf_mgr = spf_manager
+        self.fs_mgr = fs_manager
+
+    def cleanup(self) -> None:
+        self.spf_mgr.close_spf()
+        self.fs_mgr.cleanup()

testsuite/core/fs_manager.py

@@ -0,0 +1,56 @@
+import logging
+from  tempfile import TemporaryDirectory
+from pathlib import Path
+import os
+from io import StringIO
+
+class TestFSManager:
+    """Manage the temporary files for test and the cleanup
+    """    
+    def __init__(self):
+        self.logger = logging.getLogger()
+        self.logger.debug("Initialized %s", self.__class__.__name__) 
+        self.temp_dir_obj = TemporaryDirectory()
+        self.temp_dir = Path(self.temp_dir_obj.name)
+
+    def abspath(self, relative_path : Path) -> Path:
+        return self.temp_dir / relative_path
+
+    def check_exists(self, relative_path : Path) -> bool:
+        return self.abspath(relative_path).exists()
+
+    def makedirs(self, relative_path : Path) -> None:
+        # Overloaded '/' operator
+        os.makedirs(self.temp_dir / relative_path, exist_ok=True)
+
+    def create_file(self, relative_path : Path, data : str = "") -> None:
+        """Create files
+        Make sure directories exist
+        Args:
+            relative_path (Path): Relative path from test root
+        """
+        with open(self.temp_dir / relative_path, 'w', encoding="utf-8") as f:
+            f.write(data)
+
+    def tree(self, relative_root : Path = None) -> str:
+        if relative_root is None:
+            root = self.temp_dir
+        else:
+            root = self.temp_dir / relative_root
+        res = StringIO()
+        for item in root.rglob('*'):
+            path_str = str(item.relative_to(root))
+            if item.is_dir():
+                res.write(f"D-{path_str}\n")
+            else:
+                res.write(f"F-{path_str}\n")
+        return res.getvalue()
+
+    def cleanup(self) -> None:
+        """Cleaup the temporary directory
+        Its okay to forget it though, it will be cleaned on program exit then.
+        """
+        self.temp_dir_obj.cleanup()
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(temp_dir = {self.temp_dir})"