Merge pull request #94 from yoheimuta/support-extension-declarations

Support Extension Declarations for proto2 and editions

_testdata/extension_declaration.proto

@@ -0,0 +1,24 @@
+syntax = "proto2";
+
+message Foo {
+  extensions 4 to 1000 [
+    declaration = {
+      number: 4,
+      full_name: ".my.package.event_annotations",
+      type: ".logs.proto.ValidationAnnotations",
+      repeated: true },
+    declaration = {
+      number: 999,
+      full_name: ".foo.package.bar",
+      type: "int32"}];
+}
+
+message Bar {
+  extensions 1000 to 2000 [
+      declaration = {
+        number: 1000,
+        full_name: ".foo.package",
+        type: ".foo.type"
+      }
+  ];
+}

lexer/scanner/token.go

@@ -59,6 +59,10 @@ const (
 	TMAP
 	TRESERVED
 	TEXTENSIONS
+	TDECLARATION
+	TNUMBER
+	TFULLNAME
+	TTYPE
 	TENUM
 	TSTREAM
 	TGROUP
@@ -92,28 +96,32 @@ func asMiscToken(ch rune) Token {
 
 func asKeywordToken(st string) Token {
 	m := map[string]Token{
-		"syntax":     TSYNTAX,
-		"edition":    TEDITION,
-		"service":    TSERVICE,
-		"rpc":        TRPC,
-		"returns":    TRETURNS,
-		"message":    TMESSAGE,
-		"extend":     TEXTEND,
-		"import":     TIMPORT,
-		"package":    TPACKAGE,
-		"option":     TOPTION,
-		"repeated":   TREPEATED,
-		"required":   TREQUIRED,
-		"optional":   TOPTIONAL,
-		"weak":       TWEAK,
-		"public":     TPUBLIC,
-		"oneof":      TONEOF,
-		"map":        TMAP,
-		"reserved":   TRESERVED,
-		"extensions": TEXTENSIONS,
-		"enum":       TENUM,
-		"stream":     TSTREAM,
-		"group":      TGROUP,
+		"syntax":      TSYNTAX,
+		"edition":     TEDITION,
+		"service":     TSERVICE,
+		"rpc":         TRPC,
+		"returns":     TRETURNS,
+		"message":     TMESSAGE,
+		"extend":      TEXTEND,
+		"import":      TIMPORT,
+		"package":     TPACKAGE,
+		"option":      TOPTION,
+		"repeated":    TREPEATED,
+		"required":    TREQUIRED,
+		"optional":    TOPTIONAL,
+		"weak":        TWEAK,
+		"public":      TPUBLIC,
+		"oneof":       TONEOF,
+		"map":         TMAP,
+		"reserved":    TRESERVED,
+		"extensions":  TEXTENSIONS,
+		"number":      TNUMBER,
+		"full_name":   TFULLNAME,
+		"type":        TTYPE,
+		"declaration": TDECLARATION,
+		"enum":        TENUM,
+		"stream":      TSTREAM,
+		"group":       TGROUP,
 	}
 
 	if t, ok := m[st]; ok {

parser/declaration.go

@@ -0,0 +1,158 @@
+package parser
+
+import (
+	"github.com/yoheimuta/go-protoparser/v4/lexer/scanner"
+	"github.com/yoheimuta/go-protoparser/v4/parser/meta"
+)
+
+// Declaration is an option of extension ranges.
+type Declaration struct {
+	Number   string
+	FullName string
+	Type     string
+	Reserved bool
+	Repeated bool
+
+	// Comments are the optional ones placed at the beginning.
+	Comments []*Comment
+	// InlineComment is the optional one placed at the ending.
+	InlineComment *Comment
+	// InlineCommentBehindLeftCurly is the optional one placed behind a left curly.
+	InlineCommentBehindLeftCurly *Comment
+	// Meta is the meta information.
+	Meta meta.Meta
+}
+
+// SetInlineComment implements the HasInlineCommentSetter interface.
+func (d *Declaration) SetInlineComment(comment *Comment) {
+	d.InlineComment = comment
+}
+
+// Accept dispatches the call to the visitor.
+func (d *Declaration) Accept(v Visitor) {
+	if !v.VisitDeclaration(d) {
+		return
+	}
+
+	for _, comment := range d.Comments {
+		comment.Accept(v)
+	}
+	if d.InlineComment != nil {
+		d.InlineComment.Accept(v)
+	}
+	if d.InlineCommentBehindLeftCurly != nil {
+		d.InlineCommentBehindLeftCurly.Accept(v)
+	}
+}
+
+// ParseDeclaration parses a declaration.
+//
+//	declaration = "declaration" "=" "{"
+//	  "number" ":" number ","
+//	  "full_name" ":" string ","
+//	  "type" ":" string ","
+//	  "repeated" ":" bool ","
+//	  "reserved" ":" bool
+//	"}"
+//
+// See https://protobuf.dev/programming-guides/extension_declarations/
+func (p *Parser) ParseDeclaration() (*Declaration, error) {
+	p.lex.NextKeyword()
+	if p.lex.Token != scanner.TDECLARATION {
+		return nil, p.unexpected("declaration")
+	}
+	startPos := p.lex.Pos
+
+	p.lex.Next()
+	if p.lex.Token != scanner.TEQUALS {
+		return nil, p.unexpected("=")
+	}
+
+	p.lex.Next()
+	if p.lex.Token != scanner.TLEFTCURLY {
+		return nil, p.unexpected("{")
+	}
+
+	inlineLeftCurly := p.parseInlineComment()
+
+	var number string
+	var fullName string
+	var typeStr string
+	var repeated bool
+	var reserved bool
+
+	for {
+		p.lex.Next()
+		if p.lex.Token == scanner.TRIGHTCURLY {
+			break
+		}
+		if p.lex.Token != scanner.TCOMMA {
+			p.lex.UnNext()
+		}
+
+		p.lex.NextKeyword()
+		if p.lex.Token == scanner.TNUMBER {
+			p.lex.Next()
+			if p.lex.Token != scanner.TCOLON {
+				return nil, p.unexpected(":")
+			}
+			p.lex.NextNumberLit()
+			if p.lex.Token != scanner.TINTLIT {
+				return nil, p.unexpected("number")
+			}
+			number = p.lex.Text
+		} else if p.lex.Token == scanner.TFULLNAME {
+			p.lex.Next()
+			if p.lex.Token != scanner.TCOLON {
+				return nil, p.unexpected(":")
+			}
+			p.lex.NextStrLit()
+			if p.lex.Token != scanner.TSTRLIT {
+				return nil, p.unexpected("full_name string")
+			}
+			fullName = p.lex.Text
+		} else if p.lex.Token == scanner.TTYPE {
+			p.lex.Next()
+			if p.lex.Token != scanner.TCOLON {
+				return nil, p.unexpected(":")
+			}
+			p.lex.NextStrLit()
+			if p.lex.Token != scanner.TSTRLIT {
+				return nil, p.unexpected("type string")
+			}
+			typeStr = p.lex.Text
+		} else if p.lex.Token == scanner.TREPEATED {
+			p.lex.Next()
+			if p.lex.Token != scanner.TCOLON {
+				return nil, p.unexpected(":")
+			}
+			p.lex.Next()
+			if p.lex.Token != scanner.TIDENT {
+				return nil, p.unexpected("repeated bool")
+			}
+			repeated = p.lex.Text == "true"
+		} else if p.lex.Token == scanner.TRESERVED {
+			p.lex.Next()
+			if p.lex.Token != scanner.TCOLON {
+				return nil, p.unexpected(":")
+			}
+			p.lex.Next()
+			if p.lex.Token != scanner.TIDENT {
+				return nil, p.unexpected("reserved bool")
+			}
+			reserved = p.lex.Text == "true"
+		} else {
+			return nil, p.unexpected("number, full_name, type, repeated, reserved, or }")
+		}
+	}
+
+	return &Declaration{
+		Number:                       number,
+		FullName:                     fullName,
+		Type:                         typeStr,
+		Reserved:                     reserved,
+		Repeated:                     repeated,
+		InlineCommentBehindLeftCurly: inlineLeftCurly,
+		Meta:                         meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position},
+	}, nil
+}

parser/declaration_test.go

@@ -0,0 +1,105 @@
+package parser_test
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/yoheimuta/go-protoparser/v4/internal/util_test"
+	"github.com/yoheimuta/go-protoparser/v4/lexer"
+	"github.com/yoheimuta/go-protoparser/v4/parser"
+	"github.com/yoheimuta/go-protoparser/v4/parser/meta"
+)
+
+func TestParser_ParseDeclaration(t *testing.T) {
+	tests := []struct {
+		name            string
+		input           string
+		wantDeclaration *parser.Declaration
+		wantErr         bool
+	}{
+		{
+			name:    "parsing an empty",
+			wantErr: true,
+		},
+		{
+			name: "parsing an excerpt from the official reference",
+			input: `declaration = {
+      number: 4,
+      full_name: ".my.package.event_annotations",
+      type: ".logs.proto.ValidationAnnotations",
+      repeated: true }`,
+			wantDeclaration: &parser.Declaration{
+				Number:   "4",
+				FullName: `".my.package.event_annotations"`,
+				Type:     `".logs.proto.ValidationAnnotations"`,
+				Repeated: true,
+				Meta: meta.Meta{
+					Pos: meta.Position{
+						Offset: 0,
+						Line:   1,
+						Column: 1,
+					},
+					LastPos: meta.Position{
+						Offset: 153,
+						Line:   5,
+						Column: 22,
+					},
+				},
+			},
+		},
+		{
+			name: "parsing another excerpt from the official reference",
+			input: `declaration = {
+      number: 500,
+      full_name: ".my.package.event_annotations",
+      type: ".logs.proto.ValidationAnnotations",
+      reserved: true }`,
+			wantDeclaration: &parser.Declaration{
+				Number:   "500",
+				FullName: `".my.package.event_annotations"`,
+				Type:     `".logs.proto.ValidationAnnotations"`,
+				Reserved: true,
+				Meta: meta.Meta{
+					Pos: meta.Position{
+						Offset: 0,
+						Line:   1,
+						Column: 1,
+					},
+					LastPos: meta.Position{
+						Offset: 155,
+						Line:   5,
+						Column: 22,
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input)))
+			got, err := p.ParseDeclaration()
+			switch {
+			case test.wantErr:
+				if err == nil {
+					t.Errorf("got err nil, but want err")
+				}
+				return
+			case !test.wantErr && err != nil:
+				t.Errorf("got err %v, but want nil", err)
+				return
+			}
+
+			if !reflect.DeepEqual(got, test.wantDeclaration) {
+				t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantDeclaration))
+			}
+
+			if !p.IsEOF() {
+				t.Errorf("got not eof, but want eof")
+			}
+		})
+	}
+
+}