konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix another parser bug

Browse Source
This commit is contained in:
Laurenz 2022-01-31 13:44:58 +01:00
parent 8f37189d6f
commit fa57d86ed9
4 changed files with 54 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/parse/incremental.rs
View File

@ -644,6 +644,7 @@ mod tests {
test("a #let rect with (fill: eastern)\nb", 16 .. 31, " (stroke: conifer", 2 .. 34);
test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 20);
test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18);
test("#for", 4 .. 4, "//", 0 .. 6);
}
#[test]

18
src/parse/mod.rs
View File

@ -311,7 +311,7 @@ fn markup_expr(p: &mut Parser) {
p.start_group(Group::Expr);
let res = expr_prec(p, true, 0);
if stmt && res.is_ok() && !p.eof() {
p.expected_at("semicolon or line break");
p.expected("semicolon or line break");
}
p.end_group();
}
@ -435,7 +435,7 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
// Nothing.
_ => {
p.expected("expression");
p.expected_found("expression");
Err(ParseError)
}
}
@ -538,7 +538,7 @@ fn collection(p: &mut Parser) -> (CollectionKind, usize) {
items += 1;
if let Some(marker) = missing_coma.take() {
marker.expected(p, "comma");
p.expected_at(marker, "comma");
}
if p.eof() {
@ -651,7 +651,7 @@ fn block(p: &mut Parser) {
while !p.eof() {
p.start_group(Group::Expr);
if expr(p).is_ok() && !p.eof() {
p.expected_at("semicolon or line break");
p.expected("semicolon or line break");
}
p.end_group();
@ -673,7 +673,7 @@ fn args(p: &mut Parser, direct: bool, brackets: bool) -> ParseResult {
Some(NodeKind::LeftParen) => {}
Some(NodeKind::LeftBracket) if brackets => {}
_ => {
p.expected("argument list");
p.expected_found("argument list");
return Err(ParseError);
}
}
@ -726,7 +726,7 @@ fn let_expr(p: &mut Parser) -> ParseResult {
expr(p)?;
} else if has_params {
// Function definitions must have a body.
p.expected_at("body");
p.expected("body");
}
// Rewrite into a closure expression if it's a function definition.
@ -831,7 +831,7 @@ fn import_expr(p: &mut Parser) -> ParseResult {
let marker = p.marker();
let items = collection(p).1;
if items == 0 {
p.expected_at("import items");
p.expected("import items");
}
p.end_group();
@ -890,7 +890,7 @@ fn ident(p: &mut Parser) -> ParseResult {
Ok(())
}
_ => {
p.expected("identifier");
p.expected_found("identifier");
Err(ParseError)
}
}
@ -902,7 +902,7 @@ fn body(p: &mut Parser) -> ParseResult {
Some(NodeKind::LeftBracket) => template(p),
Some(NodeKind::LeftBrace) => block(p),
_ => {
p.expected_at("body");
p.expected("body");
return Err(ParseError);
}
}

88
src/parse/parser.rs
View File

@ -4,7 +4,6 @@ use std::mem;
use super::{Scanner, TokenMode, Tokens};
use crate::syntax::{ErrorPos, Green, GreenData, GreenNode, NodeKind};
use crate::util::EcoString;
/// A convenient token-based parser.
pub struct Parser<'s> {
@ -81,7 +80,7 @@ impl<'s> Parser<'s> {
Marker(self.children.len())
}
/// Create a markup right before the trailing trivia.
/// Create a marker right before the trailing trivia.
pub fn trivia_start(&self) -> Marker {
let count = self
.children
@ -155,7 +154,7 @@ impl<'s> Parser<'s> {
pub fn eat_expect(&mut self, t: &NodeKind) -> ParseResult {
let eaten = self.eat_if(t);
if !eaten {
self.expected_at(t.as_str());
self.expected(t.as_str());
}
if eaten { Ok(()) } else { Err(ParseError) }
}
@ -298,18 +297,22 @@ impl<'s> Parser<'s> {
self.stray_terminator = s;
rescan = false;
} else if required {
self.push_error(format_eco!("expected {}", end));
self.expected(end.as_str());
self.unterminated_group = true;
}
}
// Rescan the peeked token if the mode changed.
if rescan {
let mut target = self.prev_end();
if group_mode == TokenMode::Code {
self.children.truncate(self.trivia_start().0);
let start = self.trivia_start().0;
target = self.current_start
- self.children[start ..].iter().map(Green::len).sum::<usize>();
self.children.truncate(start);
}
self.tokens.jump(self.prev_end());
self.tokens.jump(target);
self.prev_end = self.tokens.index();
self.current_start = self.tokens.index();
self.current = self.tokens.next();
@ -385,42 +388,40 @@ impl<'s> Parser<'s> {
/// Error handling.
impl Parser<'_> {
/// Push an error into the children list.
pub fn push_error(&mut self, msg: impl Into<EcoString>) {
let error = NodeKind::Error(ErrorPos::Full, msg.into());
let idx = self.trivia_start();
self.children.insert(idx.0, GreenData::new(error, 0).into());
}
/// Eat the current token and add an error that it is unexpected.
pub fn unexpected(&mut self) {
match self.peek() {
Some(found) => {
let msg = format_eco!("unexpected {}", found);
let error = NodeKind::Error(ErrorPos::Full, msg);
self.perform(error, Self::eat);
}
None => self.push_error("unexpected end of file"),
if let Some(found) = self.peek() {
let msg = format_eco!("unexpected {}", found);
let error = NodeKind::Error(ErrorPos::Full, msg);
self.perform(error, Self::eat);
}
}
/// Eat the current token and add an error that it is not the expected `thing`.
/// Add an error that the `thing` was expected at the end of the last
/// non-trivia token.
pub fn expected(&mut self, thing: &str) {
self.expected_at(self.trivia_start(), thing);
}
/// Insert an error message that `what` was expected at the marker position.
pub fn expected_at(&mut self, marker: Marker, what: &str) {
let msg = format_eco!("expected {}", what);
let error = NodeKind::Error(ErrorPos::Full, msg);
self.children.insert(marker.0, GreenData::new(error, 0).into());
}
/// Eat the current token and add an error that it is not the expected
/// `thing`.
pub fn expected_found(&mut self, thing: &str) {
match self.peek() {
Some(found) => {
let msg = format_eco!("expected {}, found {}", thing, found);
let error = NodeKind::Error(ErrorPos::Full, msg);
self.perform(error, Self::eat);
}
None => self.expected_at(thing),
None => self.expected(thing),
}
}
/// Add an error that the `thing` was expected at the end of the last
/// non-trivia token.
pub fn expected_at(&mut self, thing: &str) {
self.trivia_start().expected(self, thing);
}
}
/// A marker that indicates where a node may start.
@ -428,6 +429,18 @@ impl Parser<'_> {
pub struct Marker(usize);
impl Marker {
/// Peek at the child directly after the marker.
pub fn peek<'a>(self, p: &'a Parser) -> Option<&'a Green> {
p.children.get(self.0)
}
/// Convert the child directly after marker.
pub fn convert(self, p: &mut Parser, kind: NodeKind) {
if let Some(child) = p.children.get_mut(self.0) {
child.convert(kind);
}
}
/// Perform a subparse that wraps all children after the marker in a node
/// with the given kind.
pub fn perform<T, F>(self, p: &mut Parser, kind: NodeKind, f: F) -> T
@ -471,25 +484,6 @@ impl Marker {
}
}
}
/// Insert an error message that `what` was expected at the marker position.
pub fn expected(self, p: &mut Parser, what: &str) {
let msg = format_eco!("expected {}", what);
let error = NodeKind::Error(ErrorPos::Full, msg);
p.children.insert(self.0, GreenData::new(error, 0).into());
}
/// Peek at the child directly after the marker.
pub fn peek<'a>(self, p: &'a Parser) -> Option<&'a Green> {
p.children.get(self.0)
}
/// Convert the child directly after marker.
pub fn convert(self, p: &mut Parser, kind: NodeKind) {
if let Some(child) = p.children.get_mut(self.0) {
child.convert(kind);
}
}
}
/// A logical group of tokens, e.g. `[...]`.

3
tests/typ/code/for.typ
View File

@ -94,6 +94,9 @@
// Error: 5 expected identifier
#for
// Error: 7 expected identifier
#for//
// Error: 5 expected identifier
{for}