Lexer/Parser: support decorator declaration (#556)

* Lexer/Parser: support decorator declaration

* fixing tests

* 'decorator' is a keyword now so it can't be a parameter anymore

projects/compiler/src/lexer.abra

@@ -37,6 +37,7 @@ pub enum TokenKind {
   Match
   Type
   Enum
+  Decorator
   Return(subsequentNewline: Bool)
   Readonly
   Import
@@ -111,6 +112,7 @@ pub enum TokenKind {
     TokenKind.Match => "match"
     TokenKind.Type => "type"
     TokenKind.Enum => "enum"
+    TokenKind.Decorator => "decorator"
     TokenKind.Return => "return"
     TokenKind.Readonly => "readonly"
     TokenKind.Import => "import"
@@ -664,6 +666,7 @@ pub type Lexer {
       "match" => TokenKind.Match
       "type" => TokenKind.Type
       "enum" => TokenKind.Enum
+      "decorator" => TokenKind.Decorator
       "return" => {
         val sawNewline = self._skipWhitespace()
         TokenKind.Return(sawNewline)

projects/compiler/src/parser.abra

@@ -252,6 +252,17 @@ pub type DecoratorNode {
   pub arguments: InvocationArgument[]
 }
 
+pub type DecoratorDeclarationNode {
+  pub decorators: DecoratorNode[]
+  pub pubToken: Token?
+  pub name: Label
+  pub typeParams: Label[]
+  pub fields: TypeField[]
+  pub methods: FunctionDeclarationNode[]
+  pub types: TypeDeclarationNode[]
+  pub enums: EnumDeclarationNode[]
+}
+
 pub enum AstNodeKind {
   Literal(value: LiteralAstNode)
   StringInterpolation(chunks: AstNode[])
@@ -277,6 +288,7 @@ pub enum AstNodeKind {
   FunctionDeclaration(value: FunctionDeclarationNode)
   TypeDeclaration(value: TypeDeclarationNode)
   EnumDeclaration(value: EnumDeclarationNode)
+  DecoratorDeclaration(value: DecoratorDeclarationNode)
   Break
   Continue
   Return(expr: AstNode?)
@@ -496,7 +508,7 @@ pub type Parser {
         try self._parseDecorator()
 
         val nextToken = try self._expectPeek()
-        val expected = [TokenKind.Val, TokenKind.Var, TokenKind.Func, TokenKind.Type, TokenKind.Enum, TokenKind.At, TokenKind.Pub]
+        val expected = [TokenKind.Val, TokenKind.Var, TokenKind.Func, TokenKind.Type, TokenKind.Enum, TokenKind.Decorator, TokenKind.At, TokenKind.Pub]
         if !expected.contains(nextToken.kind) {
           return Err(ParseError(position: nextToken.position, kind: ParseErrorKind.ExpectedToken(expected, nextToken.kind)))
         }
@@ -508,7 +520,7 @@ pub type Parser {
         self._advance() // consume 'pub' token
 
         val nextToken = try self._expectPeek()
-        val expected = [TokenKind.Val, TokenKind.Var, TokenKind.Func, TokenKind.Type, TokenKind.Enum]
+        val expected = [TokenKind.Val, TokenKind.Var, TokenKind.Func, TokenKind.Type, TokenKind.Enum, TokenKind.Decorator]
         if !expected.contains(nextToken.kind) {
           return Err(ParseError(position: nextToken.position, kind: ParseErrorKind.ExpectedToken(expected, nextToken.kind)))
         }
@@ -520,6 +532,7 @@ pub type Parser {
       TokenKind.Func => self._parseFunctionDeclaration()
       TokenKind.Type => self._parseTypeDeclaration()
       TokenKind.Enum => self._parseEnumDeclaration()
+      TokenKind.Decorator => self._parseDecoratorDeclaration()
       TokenKind.While => self._parseWhileLoop()
       TokenKind.For => self._parseForLoop()
       TokenKind.Break => self._parseBreak()
@@ -847,6 +860,75 @@ pub type Parser {
     Ok(AstNode(token: token, kind: AstNodeKind.EnumDeclaration(node)))
   }
 
+  func _parseDecoratorDeclaration(self): Result<AstNode, ParseError> {
+    val decorators = self._seenDecorators
+    self._seenDecorators = []
+
+    val pubToken = self._pubToken
+    self._pubToken = None
+
+    val token = try self._expectNext()
+    val decoratorName = try self._expectNextLabel()
+
+    var nextToken = try self._expectPeek()
+    val typeParams = match nextToken.kind {
+      TokenKind.LT => {
+        self._advance() // consume '<' token
+        try self._parseTypeParameters()
+      }
+      TokenKind.LBrace => []
+      _ => return Err(ParseError(position: nextToken.position, kind: ParseErrorKind.ExpectedToken([TokenKind.LParen(true)], nextToken.kind)))
+    }
+
+    try self._expectNextTokenKind(TokenKind.LBrace)
+
+    val fields: TypeField[] = []
+    var fieldPubToken: Token? = None
+    while self._peek() |nextToken| {
+      match nextToken.kind {
+        TokenKind.Ident => {
+          val field = try self._parseField(fieldPubToken)
+
+          val nextToken = try self._expectPeek()
+          if nextToken.kind == TokenKind.Comma {
+            self._advance() // consume ',' token
+          }
+
+          fields.push(field)
+          fieldPubToken = None
+        }
+        TokenKind.Pub => {
+          if fieldPubToken |token| return Err(ParseError(position: nextToken.position, kind: ParseErrorKind.ExpectedToken([TokenKind.Ident("")], nextToken.kind)))
+
+          // If the token after 'pub' is an identifier, consume 'pub'. Otherwise leave it and it'll be picked up while parsing inner decls
+          match self._peek(ahead: 1)?.kind {
+            TokenKind.Ident => self._advance()
+            else => break
+          }
+
+          fieldPubToken = Some(nextToken)
+        }
+        _ => break
+      }
+    }
+
+    val (methods, types, enums) = try self._parseBodyForTypeOrEnum()
+
+    try self._expectNextTokenKind(TokenKind.RBrace)
+
+    val node = DecoratorDeclarationNode(
+      decorators: decorators,
+      pubToken: pubToken,
+      name: decoratorName,
+      typeParams: typeParams,
+      fields: fields,
+      methods: methods,
+      types: types,
+      enums: enums,
+    )
+    Ok(AstNode(token: token, kind: AstNodeKind.DecoratorDeclaration(node)))
+  }
+
   func _parseField(self, pubToken: Token? = None): Result<TypeField, ParseError> {
     val name = try self._expectNextLabel()
     try self._expectNextTokenKind(TokenKind.Colon)

projects/compiler/src/test_utils.abra

@@ -81,6 +81,7 @@ func printTokenKindAsJson(kind: TokenKind, indentLevelStart: Int, currentIndentL
     TokenKind.Match => println("$fieldsIndent\"name\": \"Match\"")
     TokenKind.Type => println("$fieldsIndent\"name\": \"Type\"")
     TokenKind.Enum => println("$fieldsIndent\"name\": \"Enum\"")
+    TokenKind.Decorator => println("$fieldsIndent\"name\": \"Decorator\"")
     TokenKind.Return => println("$fieldsIndent\"name\": \"Return\"")
     TokenKind.Readonly => println("$fieldsIndent\"name\": \"Readonly\"")
     TokenKind.Import => println("$fieldsIndent\"name\": \"Import\"")
@@ -263,21 +264,21 @@ func printTypeIdentifierAsJson(typeIdentifier: TypeIdentifier, indentLevelStart:
   print("$endIndent}")
 }
 
-func printDecoratorNodeAsJson(decorator: DecoratorNode, indentLevelStart: Int, currentIndentLevel: Int) {
+func printDecoratorNodeAsJson(dec: DecoratorNode, indentLevelStart: Int, currentIndentLevel: Int) {
   val startIndent = "  ".repeat(indentLevelStart)
   val fieldsIndent = "  ".repeat(currentIndentLevel + 1)
   val endIndent = "  ".repeat(currentIndentLevel)
 
   print("$startIndent{\n$fieldsIndent\"name\": ")
-  printLabelAsJson(decorator.name)
+  printLabelAsJson(dec.name)
 
-  if decorator.arguments.isEmpty() {
+  if dec.arguments.isEmpty() {
     println(",\n$fieldsIndent\"arguments\": []")
   } else {
     println(",\n$fieldsIndent\"arguments\": [")
     val pathIndent = "  ".repeat(currentIndentLevel + 2)
     val pathsIndent = "  ".repeat(currentIndentLevel + 3)
-    for arg, idx in decorator.arguments {
+    for arg, idx in dec.arguments {
       print("$pathIndent{\n$pathsIndent\"label\": ")
       if arg.label |label| {
         printLabelAsJson(label)
@@ -287,7 +288,7 @@ func printDecoratorNodeAsJson(decorator: DecoratorNode, indentLevelStart: Int, c
       }
       print("$pathsIndent\"value\": ")
       printAstNodeAsJson(arg.value, 0, currentIndentLevel + 3)
-      val comma = if idx != decorator.arguments.length - 1 "," else ""
+      val comma = if idx != dec.arguments.length - 1 "," else ""
       println("\n$pathIndent}$comma")
     }
     println("$fieldsIndent]")
@@ -1125,6 +1126,104 @@ func printAstNodeKindAsJson(kind: AstNodeKind, indentLevelStart: Int, currentInd
         println("$fieldsIndent]")
       }
     }
+    AstNodeKind.DecoratorDeclaration(node) => {
+      println("$fieldsIndent\"name\": \"decoratorDeclaration\",")
+      if node.decorators.isEmpty() {
+        println("$fieldsIndent\"decorators\": [],")
+      } else {
+        println("$fieldsIndent\"decorators\": [")
+        for dec, idx in node.decorators {
+          printDecoratorNodeAsJson(dec, currentIndentLevel + 2, currentIndentLevel + 2)
+          val comma = if idx != node.decorators.length - 1 "," else ""
+          println("$comma")
+        }
+        println("$fieldsIndent],")
+      }
+      if node.pubToken |token| {
+        print("$fieldsIndent\"pubToken\": ")
+        printTokenAsJson(token, 0, currentIndentLevel + 1)
+        println(",")
+      } else {
+        println("$fieldsIndent\"pubToken\": null,")
+      }
+      print("$fieldsIndent\"typeName\": ")
+      printLabelAsJson(node.name)
+
+      if node.typeParams.isEmpty() {
+        println(",\n$fieldsIndent\"typeParams\": [],")
+      } else {
+        println(",\n$fieldsIndent\"typeParams\": [")
+        for label, idx in node.typeParams {
+          print("$fieldsIndent  ")
+          printLabelAsJson(label)
+          val comma = if idx != node.typeParams.length - 1 "," else ""
+          println("$comma")
+        }
+        println("$fieldsIndent],")
+      }
+
+      if node.fields.isEmpty() {
+        println("$fieldsIndent\"fields\": [],")
+      } else {
+        println("$fieldsIndent\"fields\": [")
+        for f, idx in node.fields {
+          println("$fieldsIndent  {")
+          print("$fieldsIndent    \"label\": ")
+          printLabelAsJson(f.name)
+          if f.pubToken |token| {
+            print(",\n$fieldsIndent    \"pubToken\": ")
+            printTokenAsJson(token, 0, currentIndentLevel + 3)
+          }
+          print(",\n$fieldsIndent    \"typeAnnotation\": ")
+          printTypeIdentifierAsJson(f.typeAnnotation, 0, currentIndentLevel + 3)
+          print(",\n$fieldsIndent    \"initializer\": ")
+          if f.initializer |initializer| {
+            printAstNodeAsJson(initializer, 0, currentIndentLevel + 3)
+          } else {
+            print("null")
+          }
+          val comma = if idx != node.fields.length - 1 "," else ""
+          println("\n$fieldsIndent  }$comma")
+        }
+        println("$fieldsIndent],")
+      }
+
+      if node.methods.isEmpty() {
+        println("$fieldsIndent\"methods\": [],")
+      } else {
+        println("$fieldsIndent\"methods\": [")
+        for method, idx in node.methods {
+          printAstNodeKindAsJson(AstNodeKind.FunctionDeclaration(method), currentIndentLevel + 2, currentIndentLevel + 2)
+          val comma = if idx != node.methods.length - 1 "," else ""
+          println("$comma")
+        }
+        println("$fieldsIndent],")
+      }
+
+      if node.types.isEmpty() {
+        println("$fieldsIndent\"nestedTypes\": [],")
+      } else {
+        println("$fieldsIndent\"nestedTypes\": [")
+        for nestedType, idx in node.types {
+          printAstNodeKindAsJson(AstNodeKind.TypeDeclaration(nestedType), currentIndentLevel + 2, currentIndentLevel + 2)
+          val comma = if idx != node.types.length - 1 "," else ""
+          println("$comma")
+        }
+        println("$fieldsIndent],")
+      }
+
+      if node.enums.isEmpty() {
+        println("$fieldsIndent\"nestedEnums\": []")
+      } else {
+        println("$fieldsIndent\"nestedEnums\": [")
+        for nestedEnum, idx in node.enums {
+          printAstNodeKindAsJson(AstNodeKind.EnumDeclaration(nestedEnum), currentIndentLevel + 2, currentIndentLevel + 2)
+          val comma = if idx != node.types.length - 1 "," else ""
+          println("$comma")
+        }
+        println("$fieldsIndent]")
+      }
+    }
     AstNodeKind.Break => println("$fieldsIndent\"name\": \"break\"")
     AstNodeKind.Continue => println("$fieldsIndent\"name\": \"continue\"")
     AstNodeKind.Return(expr) => {

projects/compiler/test/lexer/keywords.abra

@@ -1 +1 @@
-true false val var if else func while break for in type enum self match readonly import from as try continue pub
+true false val var if else func while break for in type enum self match readonly import from as try continue pub decorator

projects/compiler/test/lexer/keywords.out.json

@@ -132,5 +132,11 @@
     "kind": {
       "name": "Pub"
     }
+  },
+  {
+    "position": [1, 114],
+    "kind": {
+      "name": "Decorator"
+    }
   }
 ]

projects/compiler/test/parser/decorator_error_before_expr.out

@@ -1,4 +1,4 @@
 Error at %FILE_NAME%:2:1
-Unexpected token 'identifier', expected one of 'val', 'var', 'func', 'type', 'enum', '@', 'pub':
+Unexpected token 'identifier', expected one of 'val', 'var', 'func', 'type', 'enum', 'decorator', '@', 'pub':
   |  abc()
      ^

projects/compiler/test/parser/decorator_error_before_invalid_stmt.out

@@ -1,4 +1,4 @@
 Error at %FILE_NAME%:2:1
-Unexpected token 'while', expected one of 'val', 'var', 'func', 'type', 'enum', '@', 'pub':
+Unexpected token 'while', expected one of 'val', 'var', 'func', 'type', 'enum', 'decorator', '@', 'pub':
   |  while true { }
      ^

projects/compiler/test/parser/decoratordecl.abra

@@ -0,0 +1,60 @@
+decorator Foo { }
+decorator Foo<A, B> { }
+decorator Foo<A> { a: A, b: Bool }
+decorator Foo {
+  a: Int
+  b: Bool = true
+}
+
+decorator FooBar123 {
+  func foo(self, b: String) {}
+}
+
+decorator FooBar123 {
+  a: Int
+
+  func foo(self, b: String): X<Y> = 123
+  func bar(self, b = true) { self.a + b }
+}
+
+decorator FooBar123 {
+  a: Int
+
+  func foo(self, b: String): X<Y> = 123
+
+  @Foo
+  func bar(self, b = true) { self.a + b }
+}
+
+decorator Outer {
+  a: Int
+
+  func foo(self, b: String): X<Y> { 123 }
+
+  type InnerType {
+    a: Int
+
+    func bar(self, b = true) { self.a + b }
+  }
+
+  enum InnerEnum {
+    A(a: Int)
+
+    func bar(self, b = true) { self.a + b }
+  }
+}
+
+@Bar("a")
+pub decorator Outer {
+  a: Int
+
+  @Bar("b")
+  func foo(self, b: String): X<Y> { 123 }
+
+  @Bar("c")
+  type Inner { a: Int }
+}
+
+pub decorator Outer2 {
+  pub a: Int
+}