Added support for string tokens. OBS: parser spasser ud, men lexer virker fint. Closes: #1

Author: MartinJenzen

dictionary.py

@@ -10,6 +10,7 @@ class Dictionary:
     FLOAT                       = 'FLOAT'
     CELL                        = 'CELL'
     PLUS                        = 'PLUS'
+    STRING                      = 'STR'
     MINUS                       = 'MINUS'
     MULTIPLICATION              = 'MULT'
     DIVISION                    = 'DIV'
@@ -29,6 +30,8 @@ class Dictionary:
     LESS_THAN_OR_EQUAL_TO       = '<='
 
     multi_word_operator_parts = ['is', 'not', 'equal', 'to', 'greater', 'or', 'less', 'than']
+    
+    # TODO: Can this be optimized?
     multi_word_operators = {
         'is equal to': EQUAL_TO,
         'is not equal to': NOT_EQUAL_TO,
@@ -38,7 +41,7 @@ class Dictionary:
         'is less than or equal to': LESS_THAN_OR_EQUAL_TO
     }
 
-    operators = {
+    arithmetic_operators = {
         '+': Token(PLUS),
         '-': Token(MINUS),
         '*': Token(MULTIPLICATION),
@@ -49,6 +52,7 @@ class Dictionary:
         'is': Token(ASSIGNMENT)
     }
 
+    # TODO: Isn't this redundant?
     multi_word_operators_dictionary = {
         'is equal to': Token(EQUAL_TO),
         'is not equal to': Token(NOT_EQUAL_TO),

lexer.py

@@ -39,11 +39,14 @@ def tokenize(self):
             if self.current_character in Dictionary.NUMERIC_CHARACTERS:
                 tokens.append(self.digit_tokenize())
             # Arithmetic operators (single symbols)
-            elif self.current_character in Dictionary.operators:
-                tokens.append(Dictionary.operators[self.current_character])
+            elif self.current_character in Dictionary.arithmetic_operators:
+                tokens.append(Dictionary.arithmetic_operators[self.current_character])
             # Keywords, identifiers and multi-word arithmetic operators
             elif self.current_character in Dictionary.ALPHABETIC_CHARACTERS:
                 tokens.append(self.keyword_tokenize())
+            # Strings
+            elif self.current_character == "\"" or self.current_character == "\'":
+                tokens.append(self.string_tokenize())
             # White spaces and escape characters
             elif self.current_character in Dictionary.escape_characters:
                 tokens.append(self.escape_tokenize())
@@ -180,4 +183,19 @@ def handle_excel_cell(self):
 
     # Tokenize white spaces and escape characters (newline and tab)
     def escape_tokenize(self):
-        return Dictionary.escape_characters.get(self.current_character)
+        return Dictionary.escape_characters.get(self.current_character)
+    
+    # Generate string token (e.g. STR:'Hasta la vista, baby.' or STR:"Say hello to my little friend!")
+    def string_tokenize(self):
+        string = ""
+        quote_type = self.current_character     # Single or double quote
+        self.next_character()
+
+        while (self.current_character is not None) and (self.current_character != quote_type):
+            string += self.current_character
+            self.next_character()
+
+        # Add opening and closing quotes to string
+        string = quote_type + string + quote_type
+
+        return Token(Dictionary.STRING, string)

text_input.txt

@@ -1,2 +1,5 @@
 if 2 + 40 is equal to 42
-	x is 69
+	x is "Skynet status: online"
+	y is 'Oh shit!'
+
+cell R2D2 is 1337