Splits parsing of statements (#171)

Author: ncordon

package-lock.json

@@ -1,15 +1,15 @@
 import { CompletionItem } from 'vscode-languageserver-types';
 
 import { DbSchema } from '../dbSchema';
-import { findLatestStatement } from '../helpers';
 import { parserWrapper } from '../parserWrapper';
 import { completionCoreCompletion } from './completionCoreCompletions';
 
 export function autocomplete(
   textUntilPosition: string,
   dbSchema: DbSchema,
+  // TODO Add caret index here
 ): CompletionItem[] {
-  let parsingResult = parserWrapper.parse(textUntilPosition);
+  const parsingResult = parserWrapper.parse(textUntilPosition);
   /* We try to locate the latest statement by finding the latest available `;` 
      in the query and take from that point to the end of the query
 
@@ -22,11 +22,10 @@ export function autocomplete(
      If there was no ;, we don't want to reparse, so we return undefined 
      inside findLatestStatement
   */
-
-  const lastStatement = findLatestStatement(parsingResult);
-  if (lastStatement != undefined) {
-    parsingResult = parserWrapper.parse(lastStatement);
+  const lastStatement = parsingResult.statementsParsing.at(-1);
+  if (lastStatement !== undefined) {
+    return completionCoreCompletion(lastStatement, dbSchema);
   }
 
-  return completionCoreCompletion(parsingResult, dbSchema);
+  return [];
 }

packages/language-support/src/autocompletion/autocompletion.ts

@@ -17,11 +17,10 @@ import {
   lexerSymbols,
   tokenNames,
 } from '../lexerSymbols';
-import {
-  consoleCommandEnabled,
-  EnrichedParsingResult,
-  ParsingResult,
-} from '../parserWrapper';
+
+import { ParsedStatement } from '../parserWrapper';
+
+import { consoleCommandEnabled } from '../parserWrapper';
 
 const uniq = <T>(arr: T[]) => Array.from(new Set(arr));
 
@@ -284,7 +283,7 @@ function calculateNamespacePrefix(
 }
 
 export function completionCoreCompletion(
-  parsingResult: EnrichedParsingResult,
+  parsingResult: ParsedStatement,
   dbSchema: DbSchema,
 ): CompletionItem[] {
   const parser = parsingResult.parser;
@@ -482,7 +481,7 @@ export function completionCoreCompletion(
 }
 
 type CompletionHelperArgs = {
-  parsingResult: ParsingResult;
+  parsingResult: ParsedStatement;
   dbSchema: DbSchema;
   candidateRule: CandidateRule;
 };

packages/language-support/src/autocompletion/completionCoreCompletions.ts

@@ -12,7 +12,7 @@ import CypherParser, {
   RelationshipPatternContext,
   StatementsOrCommandsContext,
 } from './generated-parser/CypherCmdParser';
-import { ParsingResult } from './parserWrapper';
+import { ParsedStatement, ParsingResult } from './parserWrapper';
 
 /* In antlr we have 
 
@@ -92,35 +92,6 @@ type AntlrDefaultExport = {
 };
 export const antlrUtils = antlrDefaultExport as unknown as AntlrDefaultExport;
 
-export function findLatestStatement(
-  parsingResult: ParsingResult,
-): undefined | string {
-  const tokens = parsingResult.tokens;
-  const lastTokenIndex = tokens.length - 1;
-
-  let tokenIndex = lastTokenIndex;
-  let found = false;
-  let lastStatement: undefined | string = undefined;
-
-  // Last token is always EOF
-  while (tokenIndex > 0 && !found) {
-    tokenIndex--;
-    found = tokens[tokenIndex].type == CypherLexer.SEMICOLON;
-  }
-
-  if (found) {
-    lastStatement = '';
-
-    tokenIndex += 1;
-    while (tokenIndex < lastTokenIndex) {
-      lastStatement += tokens.at(tokenIndex)?.text ?? '';
-      tokenIndex++;
-    }
-  }
-
-  return lastStatement;
-}
-
 export function inNodeLabel(stopNode: ParserRuleContext) {
   const nodePattern = findParent(
     stopNode,
@@ -139,6 +110,72 @@ export function inRelationshipType(stopNode: ParserRuleContext) {
   return isDefined(relPattern);
 }
 
+export function findCaret(
+  parsingResult: ParsingResult,
+  caretPosition: number,
+): { statement: ParsedStatement; token: Token } | undefined {
+  const statements = parsingResult.statementsParsing;
+  let i = 0;
+  let result: { statement: ParsedStatement; token: Token } = undefined;
+  let keepLooking = true;
+
+  while (i < statements.length && keepLooking) {
+    let j = 0;
+    const statement = statements[i];
+    const tokens = statement.tokens;
+
+    while (j < tokens.length && keepLooking) {
+      const currentToken = tokens[j];
+      keepLooking = currentToken.start < caretPosition;
+
+      if (currentToken.channel === 0 && keepLooking) {
+        result = { statement: statement, token: currentToken };
+      }
+
+      j++;
+    }
+    i++;
+  }
+
+  return result;
+}
+
+export function splitIntoStatements(
+  tokenStream: CommonTokenStream,
+  lexer: CypherLexer,
+): CommonTokenStream[] {
+  tokenStream.fill();
+  const tokens = tokenStream.tokens;
+
+  let i = 0;
+  const result: CommonTokenStream[] = [];
+  let chunk: Token[] = [];
+  let offset = 0;
+
+  while (i < tokens.length) {
+    const current = tokens[i].clone();
+    current.tokenIndex -= offset;
+
+    chunk.push(current);
+
+    if (
+      current.type === CypherLexer.SEMICOLON ||
+      current.type === CypherLexer.EOF
+    ) {
+      // This does not relex since we are not calling fill on the token stream
+      const tokenStream = new CommonTokenStream(lexer);
+      tokenStream.tokens = chunk;
+      result.push(tokenStream);
+      offset = i + 1;
+      chunk = [];
+    }
+
+    i++;
+  }
+
+  return result;
+}
+
 export const rulesDefiningVariables = [
   CypherParser.RULE_returnItem,
   CypherParser.RULE_unwindClause,