Console command support (:use, :params, :clear, history) (#165)

* create gramamr for client commands

* some basic commands

* consider

* add test file

* mellan

* rename

* add unit tests

* cleanuptodos

* todos

* fix syntax highlighting for console commands

* add tests

* fix completions

* add errors when cmd disabled

* semantic analysis handles multiple queries

* comment

* self review

* unused variable

* fix unit tests

* self review

* rename parser

* rename rules

* rephrase comment

* fix crash and add tests

* remove todo

* mellan

* properly merge main

* improve error messages for console commands

* add changeset

* fix bad merge

Author: OskarDamkjaer

.changeset/hungry-beans-eat.md

@@ -0,0 +1,7 @@
+---
+'@neo4j-cypher/language-support': patch
+'@neo4j-cypher/react-codemirror': patch
+'@neo4j-cypher/language-server': patch
+---
+
+Add support for console commands

packages/language-server/src/linting.ts

@@ -1,8 +1,4 @@
-import {
-  findEndPosition,
-  parserWrapper,
-  validateSyntax,
-} from '@neo4j-cypher/language-support';
+import { validateSyntax } from '@neo4j-cypher/language-support';
 import debounce from 'lodash.debounce';
 import { join } from 'path';
 import { Diagnostic, TextDocumentChangeEvent } from 'vscode-languageserver';
@@ -42,9 +38,7 @@ async function rawLintDocument(
       lastSemanticJob = proxyWorker.validateSemantics(query);
       const result = await lastSemanticJob;
 
-      sendDiagnostics(
-        result.map((el) => findEndPosition(el, parserWrapper.parsingResult)),
-      );
+      sendDiagnostics(result);
     } catch (err) {
       if (!(err instanceof workerpool.Promise.CancellationError)) {
         console.error(err);

packages/language-support/package.json

@@ -47,7 +47,7 @@
     "vscode-languageserver-types": "^3.17.3"
   },
   "scripts": {
-    "gen-parser": "antlr4 -Dlanguage=TypeScript -visitor src/antlr-grammar/CypherParser.g4 src/antlr-grammar/CypherLexer.g4 -o src/generated-parser/ -Xexact-output-dir",
+    "gen-parser": "antlr4 -Dlanguage=TypeScript -visitor src/antlr-grammar/CypherCmdLexer.g4 src/antlr-grammar/CypherCmdParser.g4 -o src/generated-parser/ -Xexact-output-dir",
     "build": "npm run gen-parser && concurrently 'npm:build-types' 'npm:build-esm' 'npm:build-commonjs'",
     "build-types": "tsc --emitDeclarationOnly --outDir dist/types",
     "build-esm": "esbuild ./src/index.ts --bundle --format=esm --sourcemap --outfile=dist/esm/index.mjs",

packages/language-support/src/antlr-grammar/CypherCmdLexer.g4

@@ -0,0 +1,9 @@
+lexer grammar CypherCmdLexer;
+
+import CypherLexer;
+
+PARAM : P A R A M S?;
+
+CLEAR: C L E A R;
+
+HISTORY: H I S T O R Y;

packages/language-support/src/antlr-grammar/CypherCmdParser.g4

@@ -0,0 +1,28 @@
+parser grammar CypherCmdParser;
+
+import CypherParser;
+
+options { tokenVocab = CypherCmdLexer; }
+
+statementsOrCommands: statementOrCommand (SEMICOLON statementOrCommand)* SEMICOLON? EOF;
+
+statementOrCommand: (statement | consoleCommand);
+
+consoleCommand: COLON (clearCmd | historyCmd | useCmd | paramsCmd);
+
+paramsCmd: PARAM paramsArgs?;
+
+paramsArgs: (CLEAR | listCompletionRule | map | lambda);
+
+lambda: unescapedSymbolicNameString EQ GT expression;
+
+clearCmd: CLEAR;
+
+historyCmd: HISTORY;
+
+useCmd: useCompletionRule symbolicAliasName?;
+
+// These rules are needed to distinguish cypher <-> commands, for exapmle `USE` and `:use` in autocompletion
+listCompletionRule: LIST; 
+
+useCompletionRule: USE;

packages/language-support/src/autocompletion/completionCoreCompletions.ts

@@ -6,18 +6,22 @@ import {
   CompletionItemKind,
 } from 'vscode-languageserver-types';
 import { DbSchema } from '../dbSchema';
-import CypherLexer from '../generated-parser/CypherLexer';
+import CypherLexer from '../generated-parser/CypherCmdLexer';
 import CypherParser, {
   Expression2Context,
-} from '../generated-parser/CypherParser';
+} from '../generated-parser/CypherCmdParser';
 import { rulesDefiningVariables } from '../helpers';
 import {
   CypherTokenType,
   lexerKeywords,
   lexerSymbols,
   tokenNames,
 } from '../lexerSymbols';
-import { EnrichedParsingResult, ParsingResult } from '../parserWrapper';
+import {
+  consoleCommandEnabled,
+  EnrichedParsingResult,
+  ParsingResult,
+} from '../parserWrapper';
 
 const uniq = <T>(arr: T[]) => Array.from(new Set(arr));
 
@@ -125,17 +129,27 @@ const namespacedCompletion = (
   }
 };
 
-function getTokenCandidates(
+function getTokenCompletions(
   candidates: CandidatesCollection,
   ignoredTokens: Set<number>,
-) {
+): CompletionItem[] {
   const tokenEntries = candidates.tokens.entries();
 
-  const tokenCandidates = Array.from(tokenEntries).flatMap((value) => {
+  const completions = Array.from(tokenEntries).flatMap((value) => {
     const [tokenNumber, followUpList] = value;
 
     if (!ignoredTokens.has(tokenNumber)) {
-      const firstToken = tokenNames[tokenNumber];
+      const isConsoleCommand =
+        lexerSymbols[tokenNumber] === CypherTokenType.consoleCommand;
+
+      const kind = isConsoleCommand
+        ? CompletionItemKind.Event
+        : CompletionItemKind.Keyword;
+
+      const firstToken = isConsoleCommand
+        ? tokenNames[tokenNumber].toLowerCase()
+        : tokenNames[tokenNumber];
+
       const followUpIndexes = followUpList.indexes;
       const firstIgnoredToken = followUpIndexes.findIndex((t) =>
         ignoredTokens.has(t),
@@ -151,21 +165,28 @@ function getTokenCandidates(
       if (firstToken === undefined) {
         return [];
       } else if (followUpString === '') {
-        return [firstToken];
+        return [{ label: firstToken, kind }];
       } else {
-        const followUp = firstToken + ' ' + followUpString;
+        const followUp =
+          firstToken +
+          ' ' +
+          (isConsoleCommand ? followUpString.toLowerCase() : followUpString);
+
         if (followUpList.optional) {
-          return [firstToken, followUp];
+          return [
+            { label: firstToken, kind },
+            { label: followUp, kind },
+          ];
         }
 
-        return [followUp];
+        return [{ label: followUp, kind }];
       }
     } else {
       return [];
     }
   });
 
-  return tokenCandidates;
+  return completions;
 }
 
 const parameterCompletions = (
@@ -306,6 +327,15 @@ export function completionCoreCompletion(
     CypherParser.RULE_propertyKeyName,
     CypherParser.RULE_variable,
 
+    // Either enable the helper rules for lexer clashes,
+    // or collect all console commands like below with symbolicNameString
+    ...(consoleCommandEnabled()
+      ? [
+          CypherParser.RULE_useCompletionRule,
+          CypherParser.RULE_listCompletionRule,
+        ]
+      : [CypherParser.RULE_consoleCommand]),
+
     // Because of the overlap of keywords and identifiers in cypher
     // We will suggest keywords when users type identifiers as well
     // To avoid this we want custom completion for identifiers
@@ -317,7 +347,11 @@ export function completionCoreCompletion(
   // Keep only keywords as suggestions
   const ignoredTokens = new Set<number>(
     Object.entries(lexerSymbols)
-      .filter(([, type]) => type !== CypherTokenType.keyword)
+      .filter(
+        ([, type]) =>
+          type !== CypherTokenType.keyword &&
+          type !== CypherTokenType.consoleCommand,
+      )
       .map(([token]) => Number(token)),
   );
 
@@ -427,17 +461,24 @@ export function completionCoreCompletion(
           ];
         }
       }
+
+      // These are simple tokens that get completed as the wrong kind, due to a lexer conflict
+      if (ruleNumber === CypherParser.RULE_useCompletionRule) {
+        return [{ label: 'use', kind: CompletionItemKind.Event }];
+      }
+
+      if (ruleNumber === CypherParser.RULE_listCompletionRule) {
+        return [{ label: 'list', kind: CompletionItemKind.Event }];
+      }
+
       return [];
     },
   );
 
-  const tokenCandidates = getTokenCandidates(candidates, ignoredTokens);
-  const tokenCompletions: CompletionItem[] = tokenCandidates.map((t) => ({
-    label: t,
-    kind: CompletionItemKind.Keyword,
-  }));
-
-  return [...ruleCompletions, ...tokenCompletions];
+  return [
+    ...ruleCompletions,
+    ...getTokenCompletions(candidates, ignoredTokens),
+  ];
 }
 
 type CompletionHelperArgs = {

packages/language-support/src/helpers.ts

@@ -6,12 +6,12 @@ import antlrDefaultExport, {
   ParseTree,
   Token,
 } from 'antlr4';
-import CypherLexer from './generated-parser/CypherLexer';
+import CypherLexer from './generated-parser/CypherCmdLexer';
 import CypherParser, {
   NodePatternContext,
   RelationshipPatternContext,
-  StatementsContext,
-} from './generated-parser/CypherParser';
+  StatementsOrCommandsContext,
+} from './generated-parser/CypherCmdParser';
 import { ParsingResult } from './parserWrapper';
 
 /* In antlr we have 
@@ -31,7 +31,7 @@ export type EnrichedParseTree = ParseTree & {
   parentCtx: ParserRuleContext | undefined;
 };
 
-export function findStopNode(root: StatementsContext) {
+export function findStopNode(root: StatementsOrCommandsContext) {
   let children = root.children;
   let current: ParserRuleContext = root;
 

packages/language-support/src/highlighting/syntaxColouring/syntaxColouring.ts

@@ -5,6 +5,7 @@ import {
   AnyExpressionContext,
   ArrowLineContext,
   BooleanLiteralContext,
+  ConsoleCommandContext,
   FunctionNameContext,
   KeywordLiteralContext,
   LabelNameContext,
@@ -14,6 +15,7 @@ import {
   NoneExpressionContext,
   NumberLiteralContext,
   ParameterContext,
+  ParamsArgsContext,
   ProcedureNameContext,
   ProcedureResultItemContext,
   PropertyKeyNameContext,
@@ -23,14 +25,15 @@ import {
   StringsLiteralContext,
   StringTokenContext,
   SymbolicNameStringContext,
+  UseCompletionRuleContext,
   VariableContext,
-} from '../../generated-parser/CypherParser';
+} from '../../generated-parser/CypherCmdParser';
 
 import {
   SemanticTokensLegend,
   SemanticTokenTypes,
 } from 'vscode-languageserver-types';
-import CypherParserListener from '../../generated-parser/CypherParserListener';
+import CypherParserListener from '../../generated-parser/CypherCmdParserListener';
 import { CypherTokenType } from '../../lexerSymbols';
 import { parserWrapper } from '../../parserWrapper';
 import {
@@ -74,6 +77,7 @@ export function mapCypherToSemanticTokenIndex(
     [CypherTokenType.label]: SemanticTokenTypes.type,
     [CypherTokenType.variable]: SemanticTokenTypes.variable,
     [CypherTokenType.symbolicName]: SemanticTokenTypes.variable,
+    [CypherTokenType.consoleCommand]: SemanticTokenTypes.macro,
   };
 
   const semanticTokenType = tokenMappings[cypherTokenType];
@@ -229,6 +233,43 @@ class SyntaxHighlighter extends CypherParserListener {
   exitSymbolicNameString = (ctx: SymbolicNameStringContext) => {
     this.addToken(ctx.start, CypherTokenType.symbolicName, ctx.getText());
   };
+
+  // Fix coloring of colon in console commands (operator -> consoleCommand)
+  exitConsoleCommand = (ctx: ConsoleCommandContext) => {
+    const colon = ctx.COLON();
+    this.addToken(
+      colon.symbol,
+      CypherTokenType.consoleCommand,
+      colon.getText(),
+    );
+  };
+
+  // console commands that clash with cypher keywords
+  exitUseCompletionRule = (ctx: UseCompletionRuleContext) => {
+    const use = ctx.USE();
+
+    this.addToken(use.symbol, CypherTokenType.consoleCommand, use.getText());
+  };
+
+  exitParamsArgs = (ctx: ParamsArgsContext) => {
+    const clear = ctx.CLEAR();
+    if (clear) {
+      this.addToken(
+        clear.symbol,
+        CypherTokenType.consoleCommand,
+        clear.getText(),
+      );
+    }
+
+    const list = ctx.listCompletionRule()?.LIST();
+    if (list) {
+      this.addToken(
+        list.symbol,
+        CypherTokenType.consoleCommand,
+        list.getText(),
+      );
+    }
+  };
 }
 
 function colourLexerTokens(tokens: Token[]) {

packages/language-support/src/highlighting/syntaxColouring/syntaxColouringHelpers.ts

@@ -2,7 +2,7 @@ import { SemanticTokenTypes } from 'vscode-languageserver-types';
 
 import { Token } from 'antlr4';
 
-import CypherLexer from '../../generated-parser/CypherLexer';
+import CypherLexer from '../../generated-parser/CypherCmdLexer';
 
 import { CypherTokenType, lexerSymbols } from '../../lexerSymbols';