Add delay option, update logging, update ReadMe

testsuite/ReadMe.md

@@ -35,9 +35,38 @@ cd <superfile_root>
 ```
 
 ### Running testsuite
-- Note : You must keep your focus on the terminal for the entire duration of test run. `pyautogui` sends keypress to process on focus.
 ```
 .venv/bin/python3 main.py
 ```
 ## Setup for Windows
-Coming soon.
+Coming soon.
+
+
+
+### Python virtual env setup
+```
+# cd to this directory
+cd <path/to/here>
+python3 -m venv .venv
+.venv\Scripts\python -m pip install --upgrade pip
+.venv\Scripts\pip -r requirements.txt
+```
+
+### Make sure you build spf
+```
+# cd to the superfile repo root (parent of this)
+cd <superfile_root>
+go build -o bin/spf.exe
+```
+
+### Running testsuite
+Notes 
+- You must keep your focus on the terminal for the entire duration of test run. `pyautogui` sends keypress to process on focus.
+
+```
+.venv\Scripts\python main.py
+```
+
+## Troubleshooting failing tests
+- Use `-d` or `--debug` to enable debug logs during test run.
+- If you see flakiness in test runs due to superfile being still open, consider using `--close-wait-time` options to increase wait time for superfile to close

testsuite/core/base_test.py

@@ -66,19 +66,19 @@ def test_execute(self) -> None:
         # Start in DIR1
         self.env.spf_mgr.start_spf(self.env.fs_mgr.abspath(self.start_dir))
 
-        assert self.env.spf_mgr.is_spf_running()
+        assert self.env.spf_mgr.is_spf_running(), "Supperfile is not running"
 
         for cur_input in self.key_inputs:
             if isinstance(cur_input, keys.Keys):
                 self.env.spf_mgr.send_special_input(cur_input)
             else:
-                assert isinstance(cur_input, str)
+                assert isinstance(cur_input, str), "Invalid input type"
                 self.env.spf_mgr.send_text_input(cur_input)
             time.sleep(tconst.KEY_DELAY)
 
         time.sleep(tconst.OPERATION_DELAY)
         self.env.spf_mgr.send_special_input(keys.KEY_ESC)    
-        time.sleep(tconst.CLOSE_DELAY)
+        time.sleep(tconst.CLOSE_WAIT_TIME)
         self.logger.debug("Finished Execution")
 
     def validate(self) -> bool:
@@ -89,11 +89,11 @@ def validate(self) -> bool:
         self.logger.debug("spf_manager info : %s, Current file structure : \n%s",
             self.env.spf_mgr.runtime_info(), self.env.fs_mgr.tree(self.test_root))
         try:
-            assert not self.env.spf_mgr.is_spf_running()
+            assert not self.env.spf_mgr.is_spf_running(), "Supperfile is still running"
             for file_path in self.validation_files:
-                assert self.env.fs_mgr.check_exists(file_path)
+                assert self.env.fs_mgr.check_exists(file_path), f"File {file_path} does not exist"
         except AssertionError as ae:
-            self.logger.debug("Test assertion failed : %s", ae)
+            self.logger.debug("Test assertion failed : %s", ae, exc_info=True)
             return False
 
         return True

testsuite/core/runner.py

@@ -45,18 +45,22 @@ def run_tests(spf_path : Path, stop_on_fail : bool = True) -> None:
         cnt_passed : int = 0
         cnt_executed : int = 0
         testcases : list[BaseTest] = get_testcases(test_env)
-        logger.debug("Testcases : %s", testcases)
+        logger.info("Testcases : %s", testcases)
         for t in testcases:
+            logger.info("Running test %s", t)
             t.setup()
             t.test_execute()
             cnt_executed += 1
 
             if t.validate():
+                logger.info("Passed test %s", t)
                 cnt_passed += 1
-            elif stop_on_fail:
-                break
+            else:
+                logger.error("Failed test %s", t)
+                if stop_on_fail:
+                    break
 
-        logger.info("Finised running %s test. %s passed", cnt_executed, cnt_passed)
+        logger.info("Finished running %s test. %s passed", cnt_executed, cnt_passed)
     finally:
         # Make sure of cleanup
         # This is still not full proof, as if what happens when TestFSManager __init__ fails ?

testsuite/core/spf_manager.py

@@ -53,6 +53,7 @@ class TmuxSPFManager(BaseSPFManager):
     Tmux based Manager
     After running spf, you can connect to the session via
     tmux -L superfile attach -t spf_session
+    Wont work in windows
     """
     # Class variables
     SPF_START_DELAY : float = 0.1 # seconds
@@ -156,9 +157,19 @@ def get_rendered_output(self) -> str:
 
 
     def is_spf_running(self) -> bool:
-        self._is_spf_running = self.spf_process is not None and self.spf_process.poll() is None
+        self._is_spf_running = (self.spf_process is not None) and (self.spf_process.poll() is None)
         return self._is_spf_running
 
     def close_spf(self) -> None:
         if self.spf_process is not None:
             self.spf_process.terminate()
+
+    # Override
+    def runtime_info(self) -> str:
+        if self.spf_process is None:
+            return "[No process]"
+        else:
+            return f"[PID : {self.spf_process.pid}, poll : {self.spf_process.poll()}]"  
+
+
+

testsuite/core/test_constants.py

@@ -2,4 +2,6 @@
 
 KEY_DELAY : float       = 0.05 # seconds
 OPERATION_DELAY : float = 0.3 # seconds
-CLOSE_DELAY : float     = 0.3 # seconds
+
+# 0.3 second was too less for windows
+CLOSE_WAIT_TIME : float     = 1 # seconds

testsuite/main.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 
 from core.runner import run_tests
+import core.test_constants as tconst
 
 
 def configure_logging(debug : bool = False) -> None:
@@ -30,17 +31,20 @@ def configure_logging(debug : bool = False) -> None:
 def main():
     # Setup argument parser
     parser = argparse.ArgumentParser(description='superfile testsuite')
-    parser.add_argument('-d', '--debug',
-                        action='store_true',
+    parser.add_argument('-d', '--debug',action='store_true',
                         help='Enable debug logging')
+    parser.add_argument('--close-wait-time', type=float,
+                        help='Override default wait time after closing spf')
     parser.add_argument('--spf-path', type=str,
                         help='Override the default spf executable path(../bin/spf) under test')
 
     # Parse arguments
     args = parser.parse_args()
-
+    if args.close_wait_time is not None:
+        tconst.CLOSE_WAIT_TIME = args.close_wait_time
+
     configure_logging(args.debug)
-
+        
     # Default path
     # We maybe should run this only in main.py file.
     spf_path = Path(__file__).parent.parent / "bin" / "spf"