Enable unbounded parser lookahead

packages/langium/package.json

@@ -36,7 +36,8 @@
     "publish:latest": "npm publish --tag latest"
   },
   "dependencies": {
-    "chevrotain": "^9.1.0",
+    "chevrotain": "^10.4.1",
+    "chevrotain-allstar": "^0.1.1",
     "vscode-languageserver": "^8.0.2",
     "vscode-languageserver-textdocument": "^1.0.7",
     "vscode-uri": "^3.0.2"

packages/langium/src/parser/langium-parser.ts

@@ -5,7 +5,8 @@
  ******************************************************************************/
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
-import { defaultParserErrorProvider, DSLMethodOpts, EmbeddedActionsParser, ILexingError, IOrAlt, IParserErrorMessageProvider, IRecognitionException, IToken, TokenType, TokenVocabulary } from 'chevrotain';
+import { defaultParserErrorProvider, DSLMethodOpts, EmbeddedActionsParser, ILexingError, IOrAlt, IParserErrorMessageProvider, IRecognitionException, IToken, LLkLookaheadStrategy, TokenType, TokenVocabulary } from 'chevrotain';
+import { LLStarLookaheadStrategy } from 'chevrotain-allstar';
 import { AbstractElement, Action, Assignment, isAssignment, isCrossReference, isKeyword, ParserRule } from '../grammar/generated/ast';
 import { getTypeName, isDataTypeRule } from '../grammar/internal-grammar-util';
 import { Linker } from '../references/linker';
@@ -37,7 +38,7 @@ function isDataTypeNode(node: { $type: string | symbol | undefined }): node is D
     return node.$type === DatatypeSymbol;
 }
 
-type RuleResult = () => any;
+type RuleResult = (args: Args) => any;
 
 type Args = Record<string, boolean>;
 
@@ -150,7 +151,7 @@ export class LangiumParser extends AbstractLangiumParser {
         this.nodeBuilder.buildRootNode(input);
         const lexerResult = this.lexer.tokenize(input);
         this.wrapper.input = lexerResult.tokens;
-        const result = this.mainRule.call(this.wrapper);
+        const result = this.mainRule.call(this.wrapper, {});
         this.nodeBuilder.addHiddenTokens(lexerResult.hidden);
         this.unorderedGroups.clear();
         return {
@@ -417,7 +418,7 @@ export class LangiumCompletionParser extends AbstractLangiumParser {
         const tokens = this.lexer.tokenize(input);
         this.tokens = tokens.tokens;
         this.wrapper.input = [...this.tokens];
-        this.mainRule.call(this.wrapper);
+        this.mainRule.call(this.wrapper, {});
         this.unorderedGroups.clear();
         return {
             tokens: this.tokens,
@@ -518,9 +519,13 @@ class ChevrotainWrapper extends EmbeddedActionsParser {
     definitionErrors: IParserDefinitionError[];
 
     constructor(tokens: TokenVocabulary, config?: IParserConfig) {
+        const useDefaultLookahead = config && 'maxLookahead' in config;
         super(tokens, {
             ...defaultConfig,
-            ...config
+            lookaheadStrategy: useDefaultLookahead
+                ? new LLkLookaheadStrategy({ maxLookahead: config.maxLookahead })
+                : new LLStarLookaheadStrategy(),
+            ...config,
         });
     }
 

packages/langium/src/parser/parser-builder-base.ts

@@ -28,7 +28,7 @@ type ParserContext = {
     ruleNames: Map<AstNode, string>
 }
 
-type Rule = () => unknown;
+type Rule = (args: Args) => unknown;
 
 type Args = Record<string, boolean>;
 

packages/langium/src/validation/document-validator.ts

@@ -50,12 +50,12 @@ export class DefaultDocumentValidator implements DocumentValidator {
                 severity: DiagnosticSeverity.Error,
                 range: {
                     start: {
-                        line: lexerError.line - 1,
-                        character: lexerError.column - 1
+                        line: lexerError.line! - 1,
+                        character: lexerError.column! - 1
                     },
                     end: {
-                        line: lexerError.line - 1,
-                        character: lexerError.column + lexerError.length - 1
+                        line: lexerError.line! - 1,
+                        character: lexerError.column! + lexerError.length - 1
                     }
                 },
                 message: lexerError.message,

packages/langium/test/parser/langium-parser-builder.test.ts

@@ -19,11 +19,11 @@ describe('Predicated grammar rules with alternatives', () => {
     RuleC: 'c' TestSimple<true, false>;
     RuleD: 'd' TestSimple<false, false>;
     RuleE: 'e' TestComplex<true, true, true>;
-    RuleF: 'f' TestComplex<true, false, true>;
-    RuleG: 'g' TestComplex<false, true, false>;
+    RuleF: 'f' TestComplex<false, false, true>;
+    RuleG: 'g' TestComplex<true, false, true>;
 
-    TestSimple<A, B>: <A & B> a=ID | <B> b=ID | <A> c=ID | <!A> d=ID;
-    TestComplex<A, B, C>: <A & B & C> e=ID | <(B | C) & A> f=ID | <A | (C & false) | B> g=ID;
+    TestSimple<A, B>: <A & B> a=ID | <B & !A> b=ID | <A & !B> c=ID | <!A & !B> d=ID;
+    TestComplex<A, B, C>: <A & B & C> e=ID | <(B | C) & !A> f=ID | <(A | (C & false)) & !B> g=ID;
 
     terminal ID: '1';
     hidden terminal WS: /\\s+/;
@@ -567,6 +567,36 @@ describe('MultiMode Lexing', () => {
 
 });
 
+describe('ALL(*) parser', () => {
+
+    const grammar = `
+    grammar UnboundedLookahead
+
+    entry Entry: A | B;
+
+    // Potentially unlimited amount of 'a' tokens
+    A: {infer A} 'a'* 'b';
+    B: {infer B} 'a'* 'c';
+
+    hidden terminal WS: /\\s+/;`;
+
+    const parser = parserFromGrammar(grammar);
+
+    test('can parse with unbounded lookahead #1', () => {
+        const result = parser.parse('aaaaaaaaaab');
+        expect(result.lexerErrors).toHaveLength(0);
+        expect(result.parserErrors).toHaveLength(0);
+        expect(result.value.$type).toBe('A');
+    });
+
+    test('can parse with unbounded lookahead #2', () => {
+        const result = parser.parse('aaaaaaaaaaaaaac');
+        expect(result.lexerErrors).toHaveLength(0);
+        expect(result.parserErrors).toHaveLength(0);
+        expect(result.value.$type).toBe('B');
+    });
+});
+
 function parserFromGrammar(grammar: string): LangiumParser {
     return createServicesForGrammar({ grammar }).parser.LangiumParser;
 }