First batch of unit tests added for the lexer.py

Added a tests folder, test_lexer.py file and selected the 'unittest' framework in project settings (.vscode\settings.json).

Author: MartinJenzen

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+    "python.testing.unittestArgs": [
+        "-v",
+        "-s",
+        "./tests",
+        "-p",
+        "test_*.py"
+    ],
+    "python.testing.pytestEnabled": false,
+    "python.testing.unittestEnabled": true
+}

dictionary.py

@@ -31,7 +31,6 @@ class Dictionary:
 
     multi_word_operator_parts = ['is', 'not', 'equal', 'to', 'greater', 'or', 'less', 'than']
 
-    # TODO: Can this be optimized?
     multi_word_operators = {
         'is equal to': EQUAL_TO,
         'is not equal to': NOT_EQUAL_TO,
@@ -40,7 +39,8 @@ class Dictionary:
         'is greater than or equal to': GREATER_THAN_OR_EQUAL_TO,
         'is less than or equal to': LESS_THAN_OR_EQUAL_TO
     }
-    
+
+    # TODO: Tokenization should probably be handled within the method itself in the Lexer
     arithmetic_operators = {
         '+': Token(PLUS),
         '-': Token(MINUS),
@@ -52,7 +52,7 @@ class Dictionary:
         'is': Token(ASSIGNMENT)
     }
 
-    # TODO: Isn't this redundant?
+    # TODO: Tokenization should probably be handled within the method itself in the Lexer
     multi_word_operators_dictionary = {
         'is equal to': Token(EQUAL_TO),
         'is not equal to': Token(NOT_EQUAL_TO),
@@ -62,6 +62,7 @@ class Dictionary:
         'is less than or equal to': Token(LESS_THAN_OR_EQUAL_TO)
     }
 
+    # TODO: Tokenization should probably be handled within the method itself in the Lexer
     escape_characters = {
         '\n': Token(NEWLINE),
         '\t': Token(INDENTATION),

lexer.py

@@ -180,7 +180,9 @@ def handle_excel_cell(self):
         cell_reference = cell_reference.strip()
 
         return Token(Dictionary.CELL, cell_reference)
-    
+
+    # TODO: We should probably return the token from within this method instead of from inside the dictionary,
+    #  as well as a token value (instead of None)
     # Tokenize white spaces and escape characters (newline and tab)
     def escape_tokenize(self):
         return Dictionary.escape_characters.get(self.current_character)

tests/test_lexer.py

@@ -0,0 +1,56 @@
+# As of right now all tests can be run by either one of the following commands in the terminal:
+# python -m unittest discover -s ./tests -p test_*.py
+# python -m unittest discover -s tests
+
+import unittest
+from unittest import TestCase
+
+import lexer
+from lexer import Lexer
+
+class TestLexer(TestCase):
+    def setUp(self):
+        self.lexer = Lexer("Test")
+
+    def tearDown(self):
+        self.lexer = None
+
+    def test_current_character_in_focus_should_be_T(self):
+        self.assertEqual(self.lexer.current_character, 'T')
+
+    def test_peeked_character_should_be_e(self):
+        self.assertEqual(self.lexer.peek(), 'e')
+
+    def test_next_character_in_focus_should_be_s(self):
+        self.lexer.next_character()
+        self.lexer.next_character()
+        self.assertEqual(self.lexer.current_character, 's')
+
+    def test_advance_3_characters_should_be_t(self):
+        self.lexer.advance_n(3)
+        self.assertEqual(self.lexer.current_character, 't')
+
+    def test_peeked_word_ahead_should_be_Batman(self):
+        self.lexer = Lexer(" Batman")
+        self.assertEqual(self.lexer.peek_word_ahead(), "Batman")
+
+    def test_digit_tokenize_should_return_integer_42_token(self):
+        self.lexer = Lexer("42")
+        token = self.lexer.digit_tokenize()
+        self.assertTrue(token.type, "INTEGER")
+        self.assertEqual(token.value, 42)
+
+    def test_digit_tokenize_should_return_float_3_14_token(self):
+        self.lexer = Lexer("3.14")
+        token = self.lexer.digit_tokenize()
+        self.assertTrue(token.type, "FLOAT")
+        self.assertEqual(token.value, 3.14)
+
+    def test_escape_character_token_should_be_newline(self):
+        self.lexer = Lexer('\n')
+        token = self.lexer.escape_tokenize()
+        self.assertTrue(token.type, "NEWLINE")
+        self.assertEqual(token.value, None)
+
+if __name__ == '__main__':
+    unittest.main()