Skip to content

Commit

Permalink
Tweak: Enable PrematchByDefault so we can eliminate private keywo…
Browse files Browse the repository at this point in the history 
…rd in grammar
  • Loading branch information
@qwertie
qwertie committed Jan 8, 2017
1 parent 2afe178 commit be707e4
Show file tree
Hide file tree
Showing 4 changed files with 42 additions and 42 deletions.
44 changes: 22 additions & 22 deletions Core/Loyc.Syntax/LES/Les3Lexer.ecs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ using S = CodeSymbols;

public partial class Les3Lexer
{
[FullLLk, AddCsLineDirectives(false)]
[FullLLk, PrematchByDefault]
LLLPG (lexer) @{
// `@{` invokes LLLPG in ANTLR-style syntax mode, but there are several
// differences; see http://ecsharp.net/lllpg/lllpg-in-antlr-style.html
Expand All @@ -29,16 +29,16 @@ public partial class Les3Lexer
// Tokens with WhitespaceTag are filtered out by WhitepaceFilter.
// We want to filter out newlines inside parentheses and square brackets.
{return _brackStack.Last == TokenType.LBrace ? null : WhitespaceTag.Value;};
private token SLComment returns [object result] :
token SLComment returns [object result] :
"//" nongreedy(_)* (@"\\" | ('\r'|'\n'|EOF) =>)
{return WhitespaceTag.Value;};
[LL(3)] private token MLComment returns [object result] :
[LL(3)] token MLComment returns [object result] :
"/*" nongreedy(MLComment / Newline(true) / _)* "*/"
{return WhitespaceTag.Value;};

// Numbers ---------------------------------------------------------------

private token Number returns [object result] :
token Number returns [object result] :
('-' {_isNegative = true;} / {_isNegative = false;})
(HexNumber / BinNumber / DecNumber)
{UString numberText = Text();}
Expand All @@ -52,26 +52,26 @@ public partial class Les3Lexer
return ParseLiteral2(suffix, numberText, true);
};

private DecDigits : '0'..'9'+ greedy(('_'|'\'') '0'..'9'+)* greedy('_')? ;
DecDigits : '0'..'9'+ greedy(('_'|'\'') '0'..'9'+)* greedy('_')? ;
[inline] HexDigit : '0'..'9' | 'a'..'f' | 'A'..'F' ;
private HexDigits : greedy(HexDigit)+ greedy(('_'|'\'') greedy(HexDigit)+)* greedy('_')? ;
HexDigits : greedy(HexDigit)+ greedy(('_'|'\'') greedy(HexDigit)+)* greedy('_')? ;

// _isFloat, _numberBase are no longer used
private DecNumber :
DecNumber :
{_numberBase=10;}
(DecDigits | '.' DecDigits =>)
( {_isFloat=true;} '.' DecDigits )?
greedy( {_isFloat=true;} ('e'|'E') ('+'|'-')? DecDigits )?
;
private HexNumber() :
HexNumber :
'0' ('x'|'X') {_numberBase=16;}
(HexDigits | '.' HexDigits =>)
// Avoid ambiguity with 0x5.Equals(): a dot is not enough
( '.' ('0'..'9' => () / &( HexDigits ('p'|'P') ('+'|'-'|'0'..'9') ))
{_isFloat=true;} HexDigits )?
greedy( {_isFloat=true;} ('p'|'P') ('+'|'-')? DecDigits )?
;
private BinNumber :
BinNumber :
'0' ('b'|'B') {_numberBase=2;}
(DecDigits | '.' DecDigits =>)
( {_isFloat=true;} '.' DecDigits )?
Expand All @@ -83,38 +83,38 @@ public partial class Les3Lexer

// Strings ---------------------------------------------------------------

private token SQString returns [object result] :
token SQString returns [object result] :
{_parseNeeded = false;}
'\'' ('\\' _ {_parseNeeded = true;} | ~('\''|'\\'|'\r'|'\n')) '\''
{return ParseSQStringValue();};

private token DQString returns [object result] :
token DQString returns [object result] :
{_parseNeeded = false;}
( '"' [ '\\' _ {_parseNeeded = true;} | ~('"'|'\\'|'\r'|'\n') ]*
('"' / {_parseNeeded = true;}) )
{return ParseStringValue(isTripleQuoted: false);};

[LL(4)] private token TQString returns [object result] :
[LL(4)] token TQString returns [object result] :
{_parseNeeded = true;}
{_style = NodeStyle.TDQStringLiteral;}
( '"' '"' '"' nongreedy(Newline(true) / _)* '"' '"' '"'
| {_style = NodeStyle.TQStringLiteral;}
"'''" nongreedy(Newline(true) / _)* "'''" )
{return ParseStringValue(isTripleQuoted: true, les3TQIndents: true);};

private BQString :
BQString :
{_parseNeeded = false;}
'`' ('\\' _ {_parseNeeded = true;} | ~('`'|'\\'|'\r'|'\n'))* '`';

// Nontrivial punctuation & operators -----------------------------------

[inline] extern token OpChar :
'~'|'!'|'%'|'^'|'&'|'*'|'-'|'+'|'='|'|'|'<'|'>'|'/'|'?'|':'|'.';
private token Operator returns [object result] :
token Operator returns [object result] :
('$'|OpChar) OpChar*
{$result = ParseNormalOp();}
{if ($result == @@`':`) _type = TT.Colon;};
private token SQOperator returns [object result] :
token SQOperator returns [object result] :
"'" LettersOrPunc*
// Note: SQString has higher priority in the grammar; this rule is not called if it's a proper character
("'" {
Expand All @@ -125,15 +125,15 @@ public partial class Les3Lexer

// Identifiers, keywords and named literals --------------------------------

private token Keyword returns [object result] :
token Keyword returns [object result] :
'.'
// Check that the character before '.' is not an identifier character;
// LLLPG makes faster code in NextToken if we delay the check until after '.'
&{[Hoist] InputPosition < 2-$LI || !Try_ScanIdContChar($LI-2)}
IdStartChar+
{return (Symbol) Text();};

private token Id returns [object result] :
token Id returns [object result] :
{object boolOrNull = NoValue.Value;}
idtext:IdCore[ref boolOrNull]
( {saveAndRestore(_startPosition); _startPosition = InputPosition;}
Expand All @@ -151,9 +151,9 @@ public partial class Les3Lexer
[recognizer { bool ScanIdContChar(); }]
token IdContChar : IdStartChar | '0'..'9' | '\'' &!("''");

private NormalId : IdStartChar greedy(IdContChar)*;
NormalId : IdStartChar greedy(IdContChar)*;

private token IdCore[ref object boolOrNull] returns [UString result] :
token IdCore[ref object boolOrNull] returns [UString result] :
( BQString {$result = ParseStringCore(false);}
| NormalId {
$result = Text();
Expand All @@ -163,15 +163,15 @@ public partial class Les3Lexer
}
);

private token LettersOrPunc : '0'..'9'|'a'..'z'|'A'..'Z'|'_'|'#'
token LettersOrPunc : '0'..'9'|'a'..'z'|'A'..'Z'|'_'|'#'
|'~'|'!'|'%'|'^'|'&'|'*'|'-'|'+'|'='|'|'|'<'|'>'|'/'|'?'|':'|'.'|'$' ;

private token SpecialLiteral returns [object result] :
token SpecialLiteral returns [object result] :
"@@" LettersOrPunc+ {return ParseAtAtLiteral(Text());};

// Shebang (optional feature) --------------------------------------------

private token Shebang returns [object result] :
token Shebang returns [object result] :
&{InputPosition == 0} "#!" ~('\r'|'\n')* Newline?
{return WhitespaceTag.Value;};

Expand Down
2 changes: 1 addition & 1 deletion Core/Loyc.Syntax/LES/Les3Lexer.out.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Generated from Les3Lexer.ecs by LeMP custom tool. LeMP version: 2.4.0.1
// Generated from Les3Lexer.ecs by LeMP custom tool. LeMP version: 2.4.2.0
// Note: you can give command-line arguments to the tool via 'Custom Tool Namespace':
// --no-out-header Suppress this message
// --verbose Allow verbose messages (shown by VS as 'warnings')
Expand Down
30 changes: 15 additions & 15 deletions Core/Loyc.Syntax/LES/Les3Parser.ecs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#ecs;
#ecs;private
#importMacros(Loyc.LLPG);
using System(, .Text, .Linq, .Collections.Generic, .Diagnostics);
using Loyc; // optional (for IMessageSink, Symbol, etc.)
Expand All @@ -19,7 +19,7 @@ partial class Les3Parser
protected new const TT EOF = TT.EOF;

// Note: verbose messages are only printed when custom tool is given --verbose flag
[FullLLk, LL(2), AddCsLineDirectives(false), Verbosity(1)]
[FullLLk, LL(2), Verbosity(1), PrematchByDefault]
LLLPG (parser(laType: TT, matchType: int, terminalType: Token, allowSwitch: true)) @{

alias("@" = TT.At);
Expand Down Expand Up @@ -79,7 +79,7 @@ partial class Les3Parser
}
}

private NewlinesOpt : greedy("\n")*;
NewlinesOpt : greedy("\n")*;

// A sequence of expressions separated by commas OR semicolons.
// The `ref endMarker` parameter tells the caller if semicolons were used.
Expand Down Expand Up @@ -183,7 +183,7 @@ partial class Les3Parser
)*
) {return e;};

private InfixOperatorName[out Token op] returns [Symbol result]
InfixOperatorName[out Token op] returns [Symbol result]
: op=(TT.NormalOp|TT.Assignment|".") "\n"* {$result = (Symbol) op.Value;}
| &{[Hoist] (TT)LA($LI+1) != TT.Newline} op=":" {$result = (Symbol) op.Value;}
| &!{[Hoist] Continuators.ContainsKey(LT($LI).Value)}
Expand All @@ -208,7 +208,7 @@ partial class Les3Parser
;

// Helper rule that parses one of the syntactically special primary expressions
private FinishPrimaryExpr[LNode e] returns [LNode result]
FinishPrimaryExpr[LNode e] returns [LNode result]
: // call(function)
result:CallArgs[e]
| // Indexer / square brackets
Expand All @@ -217,15 +217,15 @@ partial class Les3Parser
{return F.Call(S.IndexBracks, args, e.Range.StartIndex, rb.EndIndex, lb.StartIndex, rb.EndIndex, NodeStyle.Operator);}
;

private CallArgs[LNode target] returns [LNode result] :
CallArgs[LNode target] returns [LNode result] :
{var endMarker = default(TokenType);}
"(" args:ExprList[ref endMarker] ")"
{
$result = F.Call(target, args, target.Range.StartIndex, $")".EndIndex).SetBaseStyle(NodeStyle.PrefixNotation);
if (endMarker == TT.Semicolon) { $result.Style |= NodeStyle.Alternate; };
};

private PrefixExpr[Precedence context] returns [LNode result]
PrefixExpr[Precedence context] returns [LNode result]
: // Prefix operator
op:(TT.NormalOp|"!"|TT.BQOperator|TT.Assignment|TT.PrefixOp|TT.PreOrSufOp)
e:Expr[PrefixPrecedenceOf(op)]
Expand All @@ -241,7 +241,7 @@ partial class Les3Parser
// - a prefix operator followed by an Expr
// - a { block } in braces
// - a [ list ] in square brackets
private Particle[bool isAttribute = false] returns [LNode result]
Particle[bool isAttribute = false] returns [LNode result]
: id:=(TT.Id|TT.BQId) // identifier
{$result = F.Id(id).SetStyle(id.Style);}
| lit:=(TT.Literal|TT.NegativeLiteral) // literal
Expand Down Expand Up @@ -275,16 +275,16 @@ partial class Les3Parser
{$result = F.Call(S.Array, list, $"[".StartIndex, $"]".EndIndex, $"[".StartIndex, $"[".EndIndex).SetStyle(NodeStyle.Expression);};

// Token lists (TODO: add unit tests)
[LL(1)]
private TokenList returns [VList<LNode> result] @init {$result = LNode.List();} :
[LL(1), #new]
TokenList returns [VList<LNode> result] @init {$result = LNode.List();} :
greedy( result+=TokenListParticle )*;
[LL(1)]
private TokenListEx returns [VList<LNode> result] :
TokenListEx returns [VList<LNode> result] :
( t:(","|";") {$result.Add(F.Id(t));}
| result+=TokenListParticle
)*;
[LL(1)]
private TokenListParticle returns [LNode result]
TokenListParticle returns [LNode result]
: "(" TokenListEx ")"
{ return F.Call(@@`'()`, $TokenListEx, $"(".StartIndex, $")".EndIndex); }
/ SquareBracketList { return $SquareBracketList; }
Expand All @@ -309,7 +309,7 @@ partial class Les3Parser
}
(=>); // Simplify output by not considering what comes afterward

private Continuator returns [LNode result] :
Continuator returns [LNode result] :
"\n"?
kw:ContinuatorKeyword {var opName = Continuators[kw.Value];}
(=>) // simplify output by preventing an unneeded check in KeywordExpression
Expand All @@ -321,12 +321,12 @@ partial class Les3Parser
/ {$result = F.Call(opName, e, kw.StartIndex, e.Range.EndIndex, kw.StartIndex, kw.EndIndex);})
);

private BracedBlock returns [LNode result] :
BracedBlock returns [LNode result] :
"{" => // simplify the output by forcing LL(1) analysis on rules using this rule
"{" stmts:=StmtList "}"
{return F.Call(S.Braces, stmts, $"{".StartIndex, $"}".EndIndex, $"{".StartIndex, $"{".EndIndex).SetStyle(NodeStyle.Statement);};

private ContinuatorKeyword returns [Token result] :
ContinuatorKeyword returns [Token result] :
&{[Hoist] Continuators.ContainsKey(LT($LI).Value)} result:TT.Id;

}; // end LLLPG parser
Expand Down
8 changes: 4 additions & 4 deletions Core/Loyc.Syntax/LES/Les3Parser.out.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Generated from Les3Parser.ecs by LeMP custom tool. LeMP version: 2.4.0.1
// Generated from Les3Parser.ecs by LeMP custom tool. LeMP version: 2.4.2.0
// Note: you can give command-line arguments to the tool via 'Custom Tool Namespace':
// --no-out-header Suppress this message
// --verbose Allow verbose messages (shown by VS as 'warnings')
Expand Down Expand Up @@ -596,15 +596,15 @@ LNode SquareBracketList()
Token lit_lsqb = default(Token);
Token lit_rsqb = default(Token);
LNode result = default(LNode);
lit_lsqb = Match((int) TT.LBrack);
lit_lsqb = MatchAny();
var list = ExprList();
lit_rsqb = Match((int) TT.RBrack);
result = F.Call(S.Array, list, lit_lsqb.StartIndex, lit_rsqb.EndIndex, lit_lsqb.StartIndex, lit_lsqb.EndIndex).SetStyle(NodeStyle.Expression);
return result;
}
static readonly HashSet<int> TokenList_set0 = NewSet((int) EOF, (int) TT.Comma, (int) TT.Newline, (int) TT.RBrace, (int) TT.RBrack, (int) TT.RParen, (int) TT.Semicolon);

VList<LNode> TokenList()
new VList<LNode> TokenList()
{
TT la0;
VList<LNode> result = default(VList<LNode>);
Expand Down Expand Up @@ -681,7 +681,7 @@ LNode KeywordExpression()
Token kw = default(Token);
LNode result = default(LNode);
var args = new VList<LNode>();
kw = Match((int) TT.Keyword);
kw = MatchAny();
// line 300
var keyword = kw.Value as Symbol;
// Line 302: ((EOF|TT.Newline|TT.RBrace|TT.RBrack|TT.RParen|TT.Semicolon) => / Expr)
Expand Down

0 comments on commit be707e4

Please sign in to comment.