Use chevrotain-allstar for lookahead

eclipse-langium · Nov 4, 2022 · 776e850 · 776e850
1 parent 4fd8469
commit 776e850
Show file tree

Hide file tree

Showing 6 changed files with 137 additions and 37 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/langium/package.json b/packages/langium/package.json
@@ -36,7 +36,8 @@
     "publish:latest": "npm publish --tag latest"
   },
   "dependencies": {
-    "chevrotain": "^9.1.0",
+    "chevrotain": "^10.4.1",
+    "chevrotain-allstar": "^0.1.1",
     "vscode-languageserver": "^8.0.2",
     "vscode-languageserver-textdocument": "^1.0.7",
     "vscode-uri": "^3.0.2"

diff --git a/packages/langium/src/parser/langium-parser.ts b/packages/langium/src/parser/langium-parser.ts
@@ -5,7 +5,8 @@
  ******************************************************************************/
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { defaultParserErrorProvider, DSLMethodOpts, EmbeddedActionsParser, ILexingError, IOrAlt, IParserErrorMessageProvider, IRecognitionException, IToken, TokenType, TokenVocabulary } from 'chevrotain';
+import { defaultParserErrorProvider, DSLMethodOpts, EmbeddedActionsParser, ILexingError, IOrAlt, IParserErrorMessageProvider, IRecognitionException, IToken, LLkLookaheadStrategy, TokenType, TokenVocabulary } from 'chevrotain';
+import { LLStarLookaheadStrategy } from 'chevrotain-allstar';
 import { AbstractElement, Action, Assignment, isAssignment, isCrossReference, isKeyword, ParserRule } from '../grammar/generated/ast';
 import { getTypeName, isDataTypeRule } from '../grammar/internal-grammar-util';
 import { Linker } from '../references/linker';
@@ -37,7 +38,7 @@ function isDataTypeNode(node: { $type: string | symbol | undefined }): node is D
     return node.$type === DatatypeSymbol;
 }
 
-type RuleResult = () => any;
+type RuleResult = (args: Args) => any;
 
 type Args = Record<string, boolean>;
 
@@ -150,7 +151,7 @@ export class LangiumParser extends AbstractLangiumParser {
         this.nodeBuilder.buildRootNode(input);
         const lexerResult = this.lexer.tokenize(input);
         this.wrapper.input = lexerResult.tokens;
-        const result = this.mainRule.call(this.wrapper);
+        const result = this.mainRule.call(this.wrapper, {});
         this.nodeBuilder.addHiddenTokens(lexerResult.hidden);
         this.unorderedGroups.clear();
         return {
@@ -417,7 +418,7 @@ export class LangiumCompletionParser extends AbstractLangiumParser {
         const tokens = this.lexer.tokenize(input);
         this.tokens = tokens.tokens;
         this.wrapper.input = [...this.tokens];
-        this.mainRule.call(this.wrapper);
+        this.mainRule.call(this.wrapper, {});
         this.unorderedGroups.clear();
         return {
             tokens: this.tokens,
@@ -518,9 +519,13 @@ class ChevrotainWrapper extends EmbeddedActionsParser {
     definitionErrors: IParserDefinitionError[];
 
     constructor(tokens: TokenVocabulary, config?: IParserConfig) {
+        const useDefaultLookahead = config && 'maxLookahead' in config;
         super(tokens, {
             ...defaultConfig,
-            ...config
+            ...config,
+            lookaheadStrategy: useDefaultLookahead
+                ? new LLkLookaheadStrategy({ maxLookahead: config.maxLookahead })
+                : new LLStarLookaheadStrategy()
         });
     }
 

diff --git a/packages/langium/src/parser/parser-builder-base.ts b/packages/langium/src/parser/parser-builder-base.ts
@@ -28,7 +28,7 @@ type ParserContext = {
     ruleNames: Map<AstNode, string>
 }
 
-type Rule = () => unknown;
+type Rule = (args: Args) => unknown;
 
 type Args = Record<string, boolean>;
 

diff --git a/packages/langium/src/validation/document-validator.ts b/packages/langium/src/validation/document-validator.ts
@@ -50,12 +50,12 @@ export class DefaultDocumentValidator implements DocumentValidator {
                 severity: DiagnosticSeverity.Error,
                 range: {
                     start: {
-                        line: lexerError.line - 1,
-                        character: lexerError.column - 1
+                        line: lexerError.line! - 1,
+                        character: lexerError.column! - 1
                     },
                     end: {
-                        line: lexerError.line - 1,
-                        character: lexerError.column + lexerError.length - 1
+                        line: lexerError.line! - 1,
+                        character: lexerError.column! + lexerError.length - 1
                     }
                 },
                 message: lexerError.message,

diff --git a/packages/langium/test/parser/langium-parser-builder.test.ts b/packages/langium/test/parser/langium-parser-builder.test.ts
@@ -567,6 +567,36 @@ describe('MultiMode Lexing', () => {
 
 });
 
+describe('ALL(*) parser', () => {
+
+    const grammar = `
+    grammar UnboundedLookahead
+
+    entry Entry: A | B;
+
+    // Potentially unlimited amount of 'a' tokens
+    A: {infer A} 'a'* 'b';
+    B: {infer B} 'a'* 'c';
+
+    hidden terminal WS: /\\s+/;`;
+
+    const parser = parserFromGrammar(grammar);
+
+    test('can parse with unbounded lookahead #1', () => {
+        const result = parser.parse('aaaaaaaaaab');
+        expect(result.lexerErrors).toHaveLength(0);
+        expect(result.parserErrors).toHaveLength(0);
+        expect(result.value.$type).toBe('A');
+    });
+
+    test('can parse with unbounded lookahead #2', () => {
+        const result = parser.parse('aaaaaaaaaaaaaac');
+        expect(result.lexerErrors).toHaveLength(0);
+        expect(result.parserErrors).toHaveLength(0);
+        expect(result.value.$type).toBe('B');
+    });
+});
+
 function parserFromGrammar(grammar: string): LangiumParser {
     return createServicesForGrammar({ grammar }).parser.LangiumParser;
 }