Skip to content

Commit

Permalink
Adds preservation of explicit newlines made by user to formatter (#382)
Browse files Browse the repository at this point in the history 
* add tests from the other branch

* good start, 3/4 for the new tests but one other test that is failing

* fix the other tests, just one remaining with commentS

* tep commit: loop over hidden tokens instead of comment tokens

* double breakline option instead, only one thing not working

* double break in preserveExplicitNewline as well

* preserve double break if we concatenate

* fix types for helpers

* add another long test

* renaming of the split choices

* refacotr determineSplits

* move the consts up

* change order of splitType

* add some GPT'ed comment explicit newline tests

* add ON MATCH ON CREATE test

* add test that wasn't idempotent and fix it

* preserve also lines after a comment

* another test

* add newline separating consts

* support explicit newlines before WHERE

* add preserving before limit as well

* add comment explaining the weird end explicit line thing
  • Loading branch information
@tomasnyberg
tomasnyberg authored Mar 5, 2025
1 parent 5145444 commit 0acc29a
Show file tree
Hide file tree
Showing 4 changed files with 659 additions and 40 deletions.
1 change: 1 addition & 0 deletions packages/language-support/src/formatting/formattingHelpersv2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export const MAX_COL = 80;

export interface BaseChunk {
isCursor?: boolean;
doubleBreak?: true;
text: string;
groupsStarting: number;
groupsEnding: number;
Expand Down
78 changes: 49 additions & 29 deletions packages/language-support/src/formatting/formattingSolutionSearchv2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const INDENTATION = 2;
const showGroups = false;

export interface Split {
splitType: ' ' | '\n' | '';
splitType: ' ' | '' | '\n' | '\n\n';
cost: number;
}

Expand Down Expand Up @@ -74,6 +74,35 @@ type FinalResult = string | FinalResultWithPos;

const openingCharacters = [CypherCmdLexer.LPAREN, CypherCmdLexer.LBRACKET];

const standardSplits: Split[] = [
{ splitType: ' ', cost: 0 },
{ splitType: '\n', cost: 1 },
];
const doubleBreakStandardSplits: Split[] = [
{ splitType: ' ', cost: 0 },
{ splitType: '\n\n', cost: 1 },
];
const noSpaceSplits: Split[] = [
{ splitType: '', cost: 0 },
{ splitType: '\n', cost: 1 },
];
const noSpaceDoubleBreakSplits: Split[] = [
{ splitType: '', cost: 0 },
{ splitType: '\n\n', cost: 1 },
];
const noBreakSplit: Split[] = [{ splitType: ' ', cost: 0 }];
const noSpaceNoBreakSplit: Split[] = [{ splitType: '', cost: 0 }];
const onlyBreakSplit: Split[] = [{ splitType: '\n', cost: 0 }];
const onlyDoubleBreakSplit: Split[] = [{ splitType: '\n\n', cost: 0 }];

const emptyChunk: RegularChunk = {
type: 'REGULAR',
text: '',
groupsStarting: 0,
groupsEnding: 0,
modifyIndentation: 0,
};

export function doesNotWantSpace(chunk: Chunk, nextChunk: Chunk): boolean {
return (
nextChunk?.type !== 'COMMENT' &&
Expand Down Expand Up @@ -110,7 +139,7 @@ function stateToString(state: State) {
}

function getNeighbourState(curr: State, choice: Choice, split: Split): State {
const isBreak = split.splitType === '\n';
const isBreak = split.splitType === '\n' || split.splitType === '\n\n';
// A state has indentation, which is applied after a hard line break. However, if it has an
// active group and we decided to split within a line, the alignment of that group takes precedence
// over the base indentation.
Expand Down Expand Up @@ -282,7 +311,10 @@ function decisionsToFormatted(decisions: Decision[]): FinalResult {
if (showGroups) addGroupEnd(buffer, decision);
buffer.push(decision.chosenSplit.splitType);
});
const result = buffer.join('').trimEnd();
let result = buffer.join('').trimEnd();
if (decisions.at(-1).left.doubleBreak) {
result += '\n';
}
if (cursorPos === -1) {
return result;
}
Expand All @@ -291,16 +323,24 @@ function decisionsToFormatted(decisions: Decision[]): FinalResult {

function determineSplits(chunk: Chunk, nextChunk: Chunk): Split[] {
if (isCommentBreak(chunk, nextChunk)) {
return onlyBreakSplit;
return chunk.doubleBreak ? onlyDoubleBreakSplit : onlyBreakSplit;
}

if (chunk.type === 'REGULAR') {
if (doesNotWantSpace(chunk, nextChunk) && chunk.noBreak)
return noSpaceNoBreakSplit;
if (doesNotWantSpace(chunk, nextChunk)) return noSpaceSplits;
if (chunk.noBreak) return noBreakSplit;
const noSpace = doesNotWantSpace(chunk, nextChunk);

if (noSpace) {
if (chunk.noBreak) {
return noSpaceNoBreakSplit;
}
return chunk.doubleBreak ? noSpaceDoubleBreakSplits : noSpaceSplits;
}
if (chunk.noBreak) {
return noBreakSplit;
}
}
return standardSplits;

return chunk.doubleBreak ? doubleBreakStandardSplits : standardSplits;
}

function chunkListToChoices(chunkList: Chunk[]): Choice[] {
Expand Down Expand Up @@ -347,23 +387,3 @@ export function buffersToFormattedString(
}
return { formattedString: formatted.trimEnd(), cursorPos: cursorPos };
}

const standardSplits: Split[] = [
{ splitType: ' ', cost: 0 },
{ splitType: '\n', cost: 1 },
];
const noSpaceSplits: Split[] = [
{ splitType: '', cost: 0 },
{ splitType: '\n', cost: 1 },
];
const noBreakSplit: Split[] = [{ splitType: ' ', cost: 0 }];
const noSpaceNoBreakSplit: Split[] = [{ splitType: '', cost: 0 }];
const onlyBreakSplit: Split[] = [{ splitType: '\n', cost: 0 }];

const emptyChunk: RegularChunk = {
type: 'REGULAR',
text: '',
groupsStarting: 0,
groupsEnding: 0,
modifyIndentation: 0,
};
103 changes: 99 additions & 4 deletions packages/language-support/src/formatting/formattingv2.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
const chunk: RegularChunk = {
type: 'REGULAR',
text: prefix.text + suffix.text,
doubleBreak: suffix.doubleBreak,
groupsStarting: prefix.groupsStarting + suffix.groupsStarting,
groupsEnding: prefix.groupsEnding + suffix.groupsEnding,
modifyIndentation: prefix.modifyIndentation + suffix.modifyIndentation,
Expand Down Expand Up @@ -180,6 +181,21 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
this.setAvoidProperty('noBreak');
};

doubleBreakBetween = (): void => {
if (this.currentBuffer().length > 0) {
this.currentBuffer().at(-1).doubleBreak = true;
}
};

doubleBreakBetweenNonComment = (): void => {
if (
this.currentBuffer().length > 0 &&
this.currentBuffer().at(-1).type !== 'COMMENT'
) {
this.currentBuffer().at(-1).doubleBreak = true;
}
};

getFirstNonCommentIdx = (): number => {
let idx = this.currentBuffer().length - 1;
while (idx >= 0 && this.currentBuffer()[idx].type === 'COMMENT') {
Expand Down Expand Up @@ -213,6 +229,53 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
this.currentBuffer().at(idx).modifyIndentation -= 1;
};

findBottomChild = (
ctx: ParserRuleContext | TerminalNode,
side: 'before' | 'after',
): TerminalNode => {
if (ctx instanceof TerminalNode) {
return ctx;
}
const idx = side === 'before' ? 0 : ctx.getChildCount() - 1;
const child = ctx.getChild(idx);
if (child instanceof TerminalNode) {
return child;
} else if (child instanceof ParserRuleContext) {
return this.findBottomChild(child, side);
}
throw new Error('Internal formatting error in findBottomChild');
};

preserveExplicitNewlineAfter = (ctx: ParserRuleContext) => {
this.preserveExplicitNewline(ctx, 'after');
};

preserveExplicitNewlineBefore = (ctx: ParserRuleContext) => {
this.preserveExplicitNewline(ctx, 'before');
};

preserveExplicitNewline = (
ctx: ParserRuleContext,
side: 'before' | 'after',
) => {
const bottomChild = this.findBottomChild(ctx, side);
const token = bottomChild.symbol;
const hiddenTokens =
side === 'before'
? this.tokenStream.getHiddenTokensToLeft(token.tokenIndex)
: this.tokenStream.getHiddenTokensToRight(token.tokenIndex);
const hiddenNewlines = hiddenTokens?.filter(
(token) => token.text === '\n',
).length;
const commentCount = hiddenTokens?.filter((token) =>
isComment(token),
).length;
// If there are comments, they take responsibility of the explicit newlines.
if (hiddenNewlines > 1 && commentCount === 0) {
this.doubleBreakBetweenNonComment();
}
};

// Comments are in the hidden channel, so grab them manually
addCommentsBefore = (node: TerminalNode) => {
const token = node.symbol;
Expand Down Expand Up @@ -243,11 +306,22 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
const hiddenTokens = this.tokenStream.getHiddenTokensToRight(
token.tokenIndex,
);
const commentTokens = (hiddenTokens || []).filter((token) =>
isComment(token),
);
const nodeLine = node.symbol.line;
for (const commentToken of commentTokens) {
let breakCount = 0;
let includesComment = false;
for (const hiddenToken of hiddenTokens || []) {
if (hiddenToken.text === '\n') {
breakCount++;
}
if (!isComment(hiddenToken)) {
continue;
}
if (breakCount > 1) {
this.doubleBreakBetween();
}
breakCount = 0;
const commentToken = hiddenToken;
includesComment = true;
const text = commentToken.text.trim();
const commentLine = commentToken.line;
const chunk: CommentChunk = {
Expand All @@ -269,6 +343,11 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
}
this.currentBuffer().push(chunk);
}
// Account for the last comment having multiple newline after it, to remember explicit
// newlines when we have e.g. [C, \n, \n]
if (breakCount > 1 && includesComment) {
this.doubleBreakBetween();
}
};

visitIfNotNull = (ctx: ParserRuleContext | TerminalNode) => {
Expand All @@ -283,11 +362,25 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
}
};

visitStatementsOrCommands = (ctx: StatementsOrCommandsContext) => {
const n = ctx.statementOrCommand_list().length;
for (let i = 0; i < n; i++) {
this.visit(ctx.statementOrCommand(i));
if (i < n - 1 || ctx.SEMICOLON(i)) {
if (this.currentBuffer().at(-1).text === '\n') {
this.currentBuffer().pop();
}
this.visit(ctx.SEMICOLON(i));
}
}
};

// Handled separately because clauses should start on new lines, see
// https://neo4j.com/docs/cypher-manual/current/styleguide/#cypher-styleguide-indentation-and-line-breaks
visitClause = (ctx: ClauseContext) => {
this.breakLine();
this.visitChildren(ctx);
this.preserveExplicitNewlineAfter(ctx);
};

visitWithClause = (ctx: WithClauseContext) => {
Expand Down Expand Up @@ -355,6 +448,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {
};

visitLimit = (ctx: LimitContext) => {
this.preserveExplicitNewlineBefore(ctx);
this.breakLine();
this.visitChildren(ctx);
};
Expand Down Expand Up @@ -776,6 +870,7 @@ export class TreePrintVisitor extends CypherCmdParserVisitor<void> {

// Handled separately because where is not a clause (it is a subclause)
visitWhereClause = (ctx: WhereClauseContext) => {
this.preserveExplicitNewlineBefore(ctx);
this.breakLine();
this.visit(ctx.WHERE());
this.avoidBreakBetween();
Expand Down
Loading

0 comments on commit 0acc29a

Please sign in to comment.