Skip to content

Commit 0d154d2

Browse files
LilyFooteLilyFoote
committed
Support lexing {% verbatim %} tags
These need special handling because they cause tags and variables within the verbatim section to be treated as text. The tests are derived from Django's test suite: https://github.com/django/django/blob/main/tests/template_tests/syntax_tests/test_verbatim.py
1 parent 65f7a70 commit 0d154d2

File tree

1 file changed

+216
-2
lines changed

1 file changed

+216
-2
lines changed

‎src/lex.rs

+216-2
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,15 @@ impl<'t> Lexer<'t> {
7070
}
7171
}
7272

73+
fn lex_text_to_end(&mut self) -> Token<'t> {
74+
let start = self.byte;
75+
let text = self.rest;
76+
self.rest = "";
77+
self.byte += text.len();
78+
let at = (start, self.byte);
79+
Token::Text { text, at }
80+
}
81+
7382
fn lex_tag(&mut self, end_tag: EndTag) -> Token<'t> {
7483
let end_str = match end_tag {
7584
EndTag::Variable => "}}",
@@ -101,7 +110,50 @@ impl<'t> Lexer<'t> {
101110
}
102111

103112
fn lex_verbatim(&mut self, verbatim: &'t str) -> Token<'t> {
104-
todo!()
113+
let verbatim = verbatim.trim();
114+
self.verbatim = None;
115+
116+
let mut rest = self.rest;
117+
let mut index = 0;
118+
let start = self.byte;
119+
loop {
120+
let next_tag = rest.find("{%");
121+
match next_tag {
122+
None => return self.lex_text_to_end(),
123+
Some(start_tag) => {
124+
rest = &rest[start_tag..];
125+
let close_tag = rest.find("%}");
126+
match close_tag {
127+
None => return self.lex_text_to_end(),
128+
Some(end_tag) => {
129+
let inner = &rest[2..end_tag].trim();
130+
// Check we have the right endverbatim tag
131+
if inner.len() < 3 || &inner[3..] != verbatim {
132+
rest = &rest[end_tag + 2..];
133+
index += start_tag + end_tag + 2;
134+
continue;
135+
}
136+
137+
index += start_tag;
138+
let text = &self.rest[..index];
139+
if text.is_empty() {
140+
// Return the endverbatim tag since we have no text
141+
let tag = &self.rest[2..end_tag];
142+
self.byte += tag.len() + 4;
143+
self.rest = &self.rest[tag.len() + 4..];
144+
let at = (start, self.byte);
145+
return Token::Tag { tag, at };
146+
} else {
147+
self.rest = &self.rest[index..];
148+
self.byte += index;
149+
let at = (start, self.byte);
150+
return Token::Text { text, at };
151+
}
152+
}
153+
}
154+
}
155+
}
156+
}
105157
}
106158
}
107159

@@ -115,7 +167,16 @@ impl<'t> Iterator for Lexer<'t> {
115167
Some(match self.verbatim {
116168
None => match self.rest.get(..START_TAG_LEN) {
117169
Some("{{") => self.lex_tag(EndTag::Variable),
118-
Some("{%") => self.lex_tag(EndTag::Tag),
170+
Some("{%") => {
171+
let tag = self.lex_tag(EndTag::Tag);
172+
if let Token::Tag { tag: verbatim, .. } = tag {
173+
let verbatim = verbatim.trim();
174+
if verbatim == "verbatim" || verbatim.starts_with("verbatim ") {
175+
self.verbatim = Some(verbatim)
176+
}
177+
}
178+
tag
179+
}
119180
Some("{#") => self.lex_tag(EndTag::Comment),
120181
_ => self.lex_text(),
121182
},
@@ -283,4 +344,157 @@ mod tests {
283344
]
284345
);
285346
}
347+
348+
#[test]
349+
fn test_verbatim_with_variable() {
350+
let template = "{% verbatim %}{{bare }}{% endverbatim %}";
351+
let lexer = Lexer::new(template);
352+
let tokens: Vec<_> = lexer.collect();
353+
assert_eq!(
354+
tokens,
355+
vec![
356+
Token::Tag {
357+
tag: " verbatim ",
358+
at: (0, 14),
359+
},
360+
Token::Text {
361+
text: "{{bare }}",
362+
at: (14, 25),
363+
},
364+
Token::Tag {
365+
tag: " endverbatim ",
366+
at: (25, 42),
367+
},
368+
]
369+
);
370+
}
371+
372+
#[test]
373+
fn test_verbatim_with_tag() {
374+
let template = "{% verbatim %}{% endif %}{% endverbatim %}";
375+
let lexer = Lexer::new(template);
376+
let tokens: Vec<_> = lexer.collect();
377+
assert_eq!(
378+
tokens,
379+
vec![
380+
Token::Tag {
381+
tag: " verbatim ",
382+
at: (0, 14),
383+
},
384+
Token::Text {
385+
text: "{% endif %}",
386+
at: (14, 25),
387+
},
388+
Token::Tag {
389+
tag: " endverbatim ",
390+
at: (25, 42),
391+
},
392+
]
393+
);
394+
}
395+
396+
#[test]
397+
fn test_verbatim_with_verbatim_tag() {
398+
let template = "{% verbatim %}It's the {% verbatim %} tag{% endverbatim %}";
399+
let lexer = Lexer::new(template);
400+
let tokens: Vec<_> = lexer.collect();
401+
assert_eq!(
402+
tokens,
403+
vec![
404+
Token::Tag {
405+
tag: " verbatim ",
406+
at: (0, 14),
407+
},
408+
Token::Text {
409+
text: "It's the {% verbatim %} tag",
410+
at: (14, 41),
411+
},
412+
Token::Tag {
413+
tag: " endverbatim ",
414+
at: (41, 58),
415+
},
416+
]
417+
);
418+
}
419+
420+
#[test]
421+
fn test_verbatim_nested() {
422+
let template = "{% verbatim %}{% verbatim %}{% endverbatim %}{% endverbatim %}";
423+
let lexer = Lexer::new(template);
424+
let tokens: Vec<_> = lexer.collect();
425+
assert_eq!(
426+
tokens,
427+
vec![
428+
Token::Tag {
429+
tag: " verbatim ",
430+
at: (0, 14),
431+
},
432+
Token::Text {
433+
text: "{% verbatim %}",
434+
at: (14, 28),
435+
},
436+
Token::Tag {
437+
tag: " endverbatim ",
438+
at: (28, 45),
439+
},
440+
Token::Tag {
441+
tag: " endverbatim ",
442+
at: (45, 62),
443+
},
444+
]
445+
);
446+
}
447+
448+
#[test]
449+
fn test_verbatim_adjacent() {
450+
let template = "{% verbatim %}{% endverbatim %}{% verbatim %}{% endverbatim %}";
451+
let lexer = Lexer::new(template);
452+
let tokens: Vec<_> = lexer.collect();
453+
assert_eq!(
454+
tokens,
455+
vec![
456+
Token::Tag {
457+
tag: " verbatim ",
458+
at: (0, 14),
459+
},
460+
Token::Tag {
461+
tag: " endverbatim ",
462+
at: (14, 31),
463+
},
464+
Token::Tag {
465+
tag: " verbatim ",
466+
at: (31, 45),
467+
},
468+
Token::Tag {
469+
tag: " endverbatim ",
470+
at: (45, 62),
471+
},
472+
]
473+
);
474+
}
475+
476+
#[test]
477+
fn test_verbatim_special() {
478+
let template =
479+
"{% verbatim special %}Don't {% endverbatim %} just yet{% endverbatim special %}";
480+
let lexer = Lexer::new(template);
481+
let tokens: Vec<_> = lexer.collect();
482+
assert_eq!(
483+
tokens,
484+
vec![
485+
Token::Tag {
486+
tag: " verbatim special ",
487+
at: (0, 22),
488+
},
489+
Token::Text {
490+
text: "Don't {% endverbatim %} just yet",
491+
at: (22, 54),
492+
},
493+
Token::Tag {
494+
tag: " endverbatim special ",
495+
at: (54, 79),
496+
},
497+
]
498+
);
499+
}
286500
}

0 commit comments

Comments
 (0)