simon/simple swift tokenizer tests

.github/workflows/test-tokenizer.yml

@@ -0,0 +1,17 @@
+name: Swift Testing
+
+on:
+  pull_request:
+    paths:
+      - 'Tokenizer/**/*.swift'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run SwiftTesting
+        uses: GetAutomaApp/opensource-actions/swifttesting@main
+        with:
+          working-directory: "Tokenizer"
+          compose: "false"

.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    "configurations": [
+        {
+            "type": "swift",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:FluxScript}/Tokenizer",
+            "name": "Debug TokenizerApp (Tokenizer)",
+            "program": "${workspaceFolder:FluxScript}/Tokenizer/.build/debug/TokenizerApp",
+            "preLaunchTask": "swift: Build Debug TokenizerApp (Tokenizer)"
+        },
+        {
+            "type": "swift",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:FluxScript}/Tokenizer",
+            "name": "Release TokenizerApp (Tokenizer)",
+            "program": "${workspaceFolder:FluxScript}/Tokenizer/.build/release/TokenizerApp",
+            "preLaunchTask": "swift: Build Release TokenizerApp (Tokenizer)"
+        }
+    ]
+}

Tokenizer/.vscode/launch.json

@@ -0,0 +1,22 @@
+{
+    "configurations": [
+        {
+            "type": "swift",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:Tokenizer}",
+            "name": "Debug TokenizerApp",
+            "program": "${workspaceFolder:Tokenizer}/.build/debug/TokenizerApp",
+            "preLaunchTask": "swift: Build Debug TokenizerApp"
+        },
+        {
+            "type": "swift",
+            "request": "launch",
+            "args": [],
+            "cwd": "${workspaceFolder:Tokenizer}",
+            "name": "Release TokenizerApp",
+            "program": "${workspaceFolder:Tokenizer}/.build/release/TokenizerApp",
+            "preLaunchTask": "swift: Build Release TokenizerApp"
+        }
+    ]
+}

Tokenizer/Sources/Tokenizer/Tokenizer.swift

@@ -8,7 +8,7 @@ public enum TokenizerError: Error {
     case invalidTokenProcessorLiteralLengthGreaterThan0
 }
 
-public enum TokenType {
+public enum TokenType: Sendable {
     case string
     case addition, subtraction, multiplication, division, modulus, exponentiation, power
     case number
@@ -17,7 +17,7 @@ public enum TokenType {
 }
 
 
-public struct Token {
+public struct Token: Equatable, Sendable {
     public let literal: String
     public let type: TokenType
     public let lineStart: Int
@@ -32,28 +32,28 @@ public class Tokenizer {
     var current: Int = 0
     var start:  Int = 0
     var line: Int = 1
-    
+
     let reservedKeywords = [
         "var": TokenType.variable
     ]
-    
+
     public init(source: String) {
         self.source = source
     }
-    
+
     public func scanTokens() throws -> [Token] {
         current = 0
         while !isAtEnd() {
             start = current
             try scanToken()
         }
-        
+
         return tokens
     }
-    
+
     func scanToken() throws {
         let literal = advance()
-        
+
         switch literal {
             case "\n":
                 line += 1
@@ -87,33 +87,33 @@ public class Tokenizer {
                 throw TokenizerError.invalidCharacter
         }
     }
-    
+
     @discardableResult
     func advance() -> Character {
         if isAtEnd() {
             return "\0"
         }
-        
+
         let index = source.index(source.startIndex, offsetBy: current)
-        
+
         current += 1
-        
+
         return source[index]
     }
-    
+
     func peek() -> Character {
         if isAtEnd() {
             return "\0"
         }
-        
+
         let index = source.index(source.startIndex, offsetBy: current)
         return source[index]
     }
-    
+
     func isAtEnd() -> Bool {
         return current >= source.count
     }
-    
+
     func addToken(type tokenType: TokenType, literal: String, startLine: Int) {
         let token = Token(
             literal: literal,
@@ -125,12 +125,12 @@ public class Tokenizer {
         )
         tokens.append(token)
     }
-    
+
     func string() throws {
         var literal: String = ""
         var isQuoted: Bool = false
         let startLine: Int = line
-        
+
         while !isAtEnd() {
             let character = advance()
             // This adds support for multiline strings
@@ -139,11 +139,11 @@ public class Tokenizer {
                 if !literal.isEmpty && !isBeforeQuoted {
                     literal += "\n"
                 }
-                
+
                 line += 1
                 continue
             }
-            
+
             // This allows us to add support for escaping characters as well!
             if character == "\\" {
                 let nextCharacter = peek()
@@ -164,15 +164,15 @@ public class Tokenizer {
                 }
                 continue
             }
-            
+
             if character != "\"" {
                 literal = "\(literal)\(character)"
             } else {
                 isQuoted.toggle()
                 break
             }
         }
-        
+
         if isQuoted {
             addToken(
                 type: .string,
@@ -183,11 +183,11 @@ public class Tokenizer {
             throw TokenizerError.unterminatedString
         }
     }
-    
+
     func isDigit(_ character: Character) -> Bool {
         return character.isNumber
     }
-    
+
     func number(first: String) throws {
         var number = first
         while !isAtEnd() {
@@ -197,47 +197,47 @@ public class Tokenizer {
                 line += 1
                 break
             }
-            
+
             if next == " " {
                 advance()
                 break
             }
-            
+
             if !isDigit(next) && next != "." {
                 break
             }
-            
+
             let current = advance()
-            
+
             if current == "." && number.contains(".") {
                 throw TokenizerError.invalidNumber
             }
-            
+
             if current == "." && isDigit(peek()) {
                 number += "."
             } else {
                 number = "\(number)\(current)"
             }
         }
-        
+
         guard Double(number) != nil else {
             throw TokenizerError.invalidNumber
         }
-        
+
         addToken(type: .number, literal: number, startLine: line)
     }
-    
+
     func identifier(first: Character) throws {
         var literal: String = "\(first)"
-        
+
         while !isAtEnd() {
             if !isAlphaNumeric(peek()) {
                 break
             }
-            
+
             literal = "\(literal)\(advance())"
         }
-        
+
         if let reservedKeywordType = reservedKeywords[literal] {
             addToken(
                 type: reservedKeywordType,
@@ -248,7 +248,7 @@ public class Tokenizer {
             addToken(type: .identifier, literal: literal, startLine: line)
         }
     }
-    
+
     func isAlphaNumeric(_ character: Character) -> Bool {
         return character.isLetter || character.isNumber
     }