diff --git a/src/macros.rs b/src/macros.rs index 5d4138701..7476de4b9 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -62,3 +62,13 @@ macro_rules! pub_use_mod { pub use $name::*; }; } + +/// Whether an expression matches a set of patterns. +macro_rules! matches { + ($expression:expr, $( $pattern:pat )|+ $( if $guard: expr )?) => { + match $expression { + $( $pattern )|+ $( if $guard )? => true, + _ => false + } + } +} diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index bcec05af5..e9725f049 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -154,6 +154,22 @@ pub struct FuncArgs { pub keyword: Object, } +#[derive(Debug, Clone, PartialEq)] +pub enum Arg { + Pos(Spanned), + Key(Pair), +} + +impl Arg { + /// The span or the value or combined span of key and value. + pub fn span(&self) -> Span { + match self { + Arg::Pos(spanned) => spanned.span, + Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + } + } +} + impl FuncArgs { pub fn new() -> FuncArgs { FuncArgs { @@ -246,32 +262,40 @@ pub struct Colorization { pub tokens: Vec>, } +impl Colorization { + pub fn new() -> Colorization { + Colorization { tokens: vec![] } + } + + pub fn add(&mut self, token: ColorToken, span: Span) { + self.tokens.push(Spanned { v: token, span }); + } + + pub fn replace_last(&mut self, token: ColorToken) { + self.tokens.last_mut().expect("replace_last: no token").v = token; + } +} + /// Entities which can be colored by syntax highlighting. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum ColorToken { Comment, - Bracket, FuncName, Colon, - Key, Equals, Comma, - Paren, Brace, - ExprIdent, ExprStr, ExprNumber, ExprSize, ExprBool, - Bold, Italic, Monospace, - Invalid, } @@ -279,3 +303,17 @@ pub enum ColorToken { pub struct ErrorMap { pub errors: Vec>, } + +impl ErrorMap { + pub fn new() -> ErrorMap { + ErrorMap { errors: vec![] } + } + + pub fn add(&mut self, message: impl Into, span: Span) { + self.errors.push(Spanned { v: message.into(), span }); + } + + pub fn add_at(&mut self, message: impl Into, pos: Position) { + self.errors.push(Spanned { v: message.into(), span: Span::at(pos) }) + } +} diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index f0a686418..71d6b251a 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -20,7 +20,6 @@ struct Parser<'s> { ctx: ParseContext<'s>, colorization: Colorization, error_map: ErrorMap, - tokens: Tokens<'s>, peeked: Option>>>, position: Position, @@ -32,9 +31,8 @@ impl<'s> Parser<'s> { Parser { src, ctx, - error_map: ErrorMap { errors: vec![] }, - colorization: Colorization { tokens: vec![] }, - + error_map: ErrorMap::new(), + colorization: Colorization::new(), tokens: Tokens::new(src), peeked: None, position: Position::ZERO, @@ -42,6 +40,7 @@ impl<'s> Parser<'s> { } } + /// The main parsing entrypoint. fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) { let mut tree = SyntaxTree::new(); @@ -79,10 +78,18 @@ impl<'s> Parser<'s> { (tree, self.colorization, self.error_map) } + /// Parses a function including header and body with the cursor starting + /// right behind the first opening bracket. fn parse_func(&mut self) -> Option> { let start = self.last_pos(); let header = self.parse_func_header(); + self.eat_until(|t| t == RightBracket, false); + + if self.eat().map(Spanned::value) != Some(RightBracket) { + self.expected_at("closing bracket", self.pos()); + } + let call = self.parse_func_call(header)?; let end = self.pos(); @@ -91,21 +98,17 @@ impl<'s> Parser<'s> { Some(Spanned { v: Node::Func(call), span }) } + /// Parses a function header including the closing bracket. fn parse_func_header(&mut self) -> Option { self.skip_whitespace(); - - let name = self.parse_func_name().or_else(|| { - self.eat_until(|t| t == RightBracket, true); - None - })?; + let name = self.parse_func_name()?; self.skip_whitespace(); let args = match self.eat() { Some(Spanned { v: Colon, .. }) => self.parse_func_args(), Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(), other => { - self.expected("colon or closing bracket", other); - self.eat_until(|t| t == RightBracket, true); + self.expected_at("colon or closing bracket", name.span.end); FuncArgs::new() } }; @@ -113,8 +116,140 @@ impl<'s> Parser<'s> { Some(FuncHeader { name, args }) } + /// Parses the function name if is the next token. Otherwise, it adds an + /// error and returns `None`. + fn parse_func_name(&mut self) -> Option> { + match self.eat() { + Some(Spanned { v: ExprIdent(ident), span }) => { + self.colorization.replace_last(ColorToken::FuncName); + return Some(Spanned { v: Ident(ident.to_string()), span }); + } + other => self.expected_found_or_at("identifier", other, self.pos()), + } + + None + } + + /// Parses the function arguments and stops right before the final closing + /// bracket. + fn parse_func_args(&mut self) -> FuncArgs { + let mut args = FuncArgs::new(); + + loop { + self.skip_whitespace(); + match self.peekv() { + Some(RightBracket) | None => break, + _ => match self.parse_arg() { + Some(Arg::Pos(item)) => args.add_pos(item), + Some(Arg::Key(pair)) => args.add_key_pair(pair), + None => {} + } + } + } + + args + } + + /// Parse a positional or keyword argument. + fn parse_arg(&mut self) -> Option { + let first = self.peek()?; + let span = first.span; + + let arg = if let ExprIdent(ident) = first.v { + self.eat(); + self.skip_whitespace(); + + let ident = Ident(ident.to_string()); + if let Some(Equals) = self.peekv() { + self.colorization.replace_last(ColorToken::Key); + + self.eat(); + self.skip_whitespace(); + + self.parse_expr().map(|value| { + Arg::Key(Pair { + key: Spanned { v: ident, span }, + value, + }) + }) + } else { + Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span))) + } + } else { + self.parse_expr().map(|expr| Arg::Pos(expr)) + }; + + if let Some(arg) = &arg { + self.skip_whitespace(); + match self.peekv() { + Some(RightBracket) => {} + Some(Comma) => { self.eat(); } + Some(_) => self.expected_at("comma", arg.span().end), + _ => {} + } + } else { + let found = self.eat(); + self.expected_found_or_at("value", found, self.pos()); + } + + arg + } + + /// Parse a atomic or compound (tuple / object) expression. + fn parse_expr(&mut self) -> Option> { + let first = self.peek()?; + let mut expr = |v| { + self.eat(); + Spanned { v, span: first.span } + }; + + Some(match first.v { + ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))), + ExprStr(s) => expr(Expression::Str(s.to_string())), + ExprNumber(n) => expr(Expression::Number(n)), + ExprSize(s) => expr(Expression::Size(s)), + ExprBool(b) => expr(Expression::Bool(b)), + LeftParen => self.parse_tuple(), + LeftBrace => self.parse_object(), + _ => return None, + }) + } + + /// Parse a tuple expression. + fn parse_tuple(&mut self) -> Spanned { + let start = self.pos(); + + // TODO: Do the thing. + self.eat_until(|t| matches!(t, RightParen | RightBracket), false); + if self.peekv() == Some(RightParen) { + self.eat(); + } + + let end = self.pos(); + let span = Span { start, end }; + + Spanned { v: Expression::Tuple(Tuple::new()), span } + } + + /// Parse an object expression. + fn parse_object(&mut self) -> Spanned { + let start = self.pos(); + + // TODO: Do the thing. + self.eat_until(|t| matches!(t, RightBrace | RightBracket), false); + if self.peekv() == Some(RightBrace) { + self.eat(); + } + + let end = self.pos(); + let span = Span { start, end }; + + Spanned { v: Expression::Object(Object::new()), span } + } + + /// Parse the body of a function invocation. fn parse_func_call(&mut self, header: Option) -> Option { - let body = if self.peek() == Some(LeftBracket) { + let body = if self.peekv() == Some(LeftBracket) { self.eat(); let start = self.tokens.index(); @@ -127,9 +262,10 @@ impl<'s> Parser<'s> { let body = &self.src[start .. end]; if found { - assert_eq!(self.eat().map(Spanned::value), Some(RightBracket)); + let next = self.eat().map(Spanned::value); + debug_assert_eq!(next, Some(RightBracket)); } else { - self.error_here("expected closing bracket"); + self.expected_at("closing bracket", self.pos()); } Some(body) @@ -139,111 +275,57 @@ impl<'s> Parser<'s> { let header = header?; let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| { - self.error( - format!("unknown function: `{}`", header.name.v), - header.name.span - ); + let message = format!("unknown function: `{}`", header.name.v); + self.error_map.add(message, header.name.span); None })?; Some(FuncCall(parser(header, body, self.ctx).unwrap())) } - fn parse_func_name(&mut self) -> Option> { - match self.eat() { - Some(Spanned { v: ExprIdent(ident), span }) => { - self.color(Spanned { v: ColorToken::FuncName, span }, true); - Some(Spanned { v: Ident(ident.to_string()), span }) - } - other => { - self.expected("identifier", other); - None - } - } - } - - fn parse_func_args(&mut self) -> FuncArgs { - // todo!() - self.eat_until(|t| t == RightBracket, true); - FuncArgs::new() - } - - fn parse_tuple(&mut self) -> Spanned { - todo!("parse_tuple") - } - - fn parse_object(&mut self) -> Spanned { - todo!("parse_object") - } - + /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { - self.eat_until(|t| match t { - Whitespace(_) | LineComment(_) | BlockComment(_) => false, - _ => true, - }, false) + self.eat_until(|t| + !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false) } - fn expected(&mut self, thing: &str, found: Option>) { - if let Some(Spanned { v: found, span }) = found { - self.error( - format!("expected {}, found {}", thing, name(found)), - span - ); - } else { - self.error_here(format!("expected {}", thing)); + /// Add an error about an `thing` which was expected but not found at the + /// given position. + fn expected_at(&mut self, thing: &str, pos: Position) { + self.error_map.add_at(format!("expected {}", thing), pos); + } + + /// Add an error about an expected `thing` which was not found, showing + /// what was found instead. + fn expected_found(&mut self, thing: &str, found: Spanned) { + let message = format!("expected {}, found {}", thing, name(found.v)); + self.error_map.add(message, found.span); + } + + /// Add a found-error if `found` is some and a positional error, otherwise. + fn expected_found_or_at( + &mut self, + thing: &str, + found: Option>, + pos: Position + ) { + match found { + Some(found) => self.expected_found(thing, found), + None => self.expected_at(thing, pos), } } + /// Add an error about an unexpected token `found`. fn unexpected(&mut self, found: Spanned) { - self.error_map.errors.push(found.map(|t| format!("unexpected {}", name(t)))); - } - - fn error(&mut self, message: impl Into, span: Span) { - self.error_map.errors.push(Spanned { v: message.into(), span }); - } - - fn error_here(&mut self, message: impl Into) { - self.error(message, Span::at(self.pos())); - } - - fn color(&mut self, token: Spanned, replace_last: bool) { - if replace_last { - if let Some(last) = self.colorization.tokens.last_mut() { - *last = token; - return; - } - } - - self.colorization.tokens.push(token); - } - - fn color_token(&mut self, token: Spanned>) { - let colored = match token.v { - LineComment(_) | BlockComment(_) => Some(ColorToken::Comment), - StarSlash => Some(ColorToken::Invalid), - LeftBracket | RightBracket => Some(ColorToken::Bracket), - LeftParen | RightParen => Some(ColorToken::Paren), - LeftBrace | RightBrace => Some(ColorToken::Brace), - Colon => Some(ColorToken::Colon), - Comma => Some(ColorToken::Comma), - Equals => Some(ColorToken::Equals), - ExprIdent(_) => Some(ColorToken::ExprIdent), - ExprStr(_) => Some(ColorToken::ExprStr), - ExprNumber(_) => Some(ColorToken::ExprNumber), - ExprSize(_) => Some(ColorToken::ExprSize), - ExprBool(_) => Some(ColorToken::ExprBool), - _ => None, - }; - - if let Some(color) = colored { - self.colorization.tokens.push(Spanned { v: color, span: token.span }); - } + self.error_map.add(format!("unexpected {}", name(found.v)), found.span); } + /// Consume tokens until the function returns true and only consume the last + /// token if instructed to. fn eat_until(&mut self, mut f: F, eat_match: bool) where F: FnMut(Token<'s>) -> bool { while let Some(token) = self.peek() { - if f(token) { + if f(token.v) { if eat_match { self.eat(); } @@ -254,56 +336,90 @@ impl<'s> Parser<'s> { } } + /// Consume and return the next token, update positions and colorize the + /// token. All colorable tokens are per default colorized here, to override + /// a colorization use `Colorization::replace_last`. fn eat(&mut self) -> Option>> { - let token = self.peeked.take().unwrap_or_else(|| self.tokens.next()); + let token = self.peeked.take() + .unwrap_or_else(|| self.tokens.next()); - self.last_position = self.position; - if let Some(spanned) = token { - self.color_token(spanned); - self.position = spanned.span.end; + if let Some(token) = token { + if let Some(color) = color(token.v) { + self.colorization.add(color, token.span); + } + + self.last_position = self.position; + self.position = token.span.end; } token } - fn peek(&mut self) -> Option> { + /// Peek at the next token without consuming it. + fn peek(&mut self) -> Option>> { let iter = &mut self.tokens; - self.peeked - .get_or_insert_with(|| iter.next()) - .map(Spanned::value) + *self.peeked.get_or_insert_with(|| iter.next()) } + fn peekv(&mut self) -> Option> { + self.peek().map(Spanned::value) + } + + /// The position at the end of the last eat token / start of the peekable + /// token. fn pos(&self) -> Position { self.position } + /// The position at the start of the last eaten token. fn last_pos(&self) -> Position { self.last_position } } +/// The name of a token in an `expected <...>` error. fn name(token: Token) -> &'static str { match token { Whitespace(_) => "whitespace", LineComment(_) | BlockComment(_) => "comment", - StarSlash => "end of block comment", - LeftBracket => "opening bracket", - RightBracket => "closing bracket", - LeftParen => "opening paren", - RightParen => "closing paren", - LeftBrace => "opening brace", - RightBrace => "closing brace", - Colon => "colon", - Comma => "comma", - Equals => "equals sign", - ExprIdent(_) => "identifier", - ExprStr(_) => "string", + StarSlash => "end of block comment", + LeftBracket => "opening bracket", + RightBracket => "closing bracket", + LeftParen => "opening paren", + RightParen => "closing paren", + LeftBrace => "opening brace", + RightBrace => "closing brace", + Colon => "colon", + Comma => "comma", + Equals => "equals sign", + ExprIdent(_) => "identifier", + ExprStr(_) => "string", ExprNumber(_) => "number", - ExprSize(_) => "size", - ExprBool(_) => "bool", - Star => "star", - Underscore => "underscore", - Backtick => "backtick", - Text(_) => "text", + ExprSize(_) => "size", + ExprBool(_) => "bool", + Star => "star", + Underscore => "underscore", + Backtick => "backtick", + Text(_) => "invalid identifier", } } + +/// The color token corresponding to a token. +fn color(token: Token) -> Option { + Some(match token { + LineComment(_) | BlockComment(_) => ColorToken::Comment, + LeftBracket | RightBracket => ColorToken::Bracket, + LeftParen | RightParen => ColorToken::Paren, + LeftBrace | RightBrace => ColorToken::Brace, + Colon => ColorToken::Colon, + Comma => ColorToken::Comma, + Equals => ColorToken::Equals, + ExprIdent(_) => ColorToken::ExprIdent, + ExprStr(_) => ColorToken::ExprStr, + ExprNumber(_) => ColorToken::ExprNumber, + ExprSize(_) => ColorToken::ExprSize, + ExprBool(_) => ColorToken::ExprBool, + StarSlash => ColorToken::Invalid, + _ => return None, + }) +} diff --git a/tests/parser/trees.rs b/tests/parser/trees.rs index 442f71dd7..d761fe249 100644 --- a/tests/parser/trees.rs +++ b/tests/parser/trees.rs @@ -1,15 +1,19 @@ +// Basics. p "" => [] p "hi" => [T("hi")] p "hi you" => [T("hi"), S, T("you")] p "❤\n\n 🌍" => [T("❤"), N, T("🌍")] +// Functions. p "[func]" => [func!("func"; None)] p "[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, B, T("you"), B]))] - p "from [align: left] to" => [ T("from"), S, func!("align", pos: [ID("left")]; None), S, T("to"), ] - +p "[f: left, 12pt, false]" => [ + func!("f", pos: [ID("left"), SIZE(Size::pt(12.0)), BOOL(false)]; None) +] +p "[f: , hi, * \"du\"]" => [func!("f", pos: [ID("hi"), STR("du")]; None)] p "[box: x=1.2pt, false][a b c] bye" => [ func!( "box", @@ -20,14 +24,23 @@ p "[box: x=1.2pt, false][a b c] bye" => [ S, T("bye"), ] -c "hi" => [] -c "[align: left][\n _body_\n]" => [ - (0:0, 0:1, B), - (0:1, 0:6, FN), - (0:6, 0:7, CL), - (0:8, 0:12, ID), - (0:12, 0:13, B), - (0:13, 0:14, B), - (1:4, 1:10, IT), - (2:0, 2:2, B), +// Errors. +e "[f: , hi, * \"du\"]" => [ + (0:4, 0:5, "expected value, found comma"), + (0:10, 0:11, "expected value, found invalid identifier"), +] +e "[f:, , ,]" => [ + (0:3, 0:4, "expected value, found comma"), + (0:5, 0:6, "expected value, found comma"), + (0:7, 0:8, "expected value, found comma"), +] +e "[f:" => [(0:3, 0:3, "expected closing bracket")] +e "[f: hi" => [(0:6, 0:6, "expected closing bracket")] +e "[f: hey 12pt]" => [(0:7, 0:7, "expected comma")] +e "[box: x=, false y=z=4" => [ + (0:8, 0:9, "expected value, found comma"), + (0:15, 0:15, "expected comma"), + (0:19, 0:19, "expected comma"), + (0:19, 0:20, "expected value, found equals sign"), + (0:21, 0:21, "expected closing bracket"), ] diff --git a/tests/src/parser.rs b/tests/src/parser.rs index ecf1544c3..b2aa01da9 100644 --- a/tests/src/parser.rs +++ b/tests/src/parser.rs @@ -40,7 +40,7 @@ fn test(tests: Vec<(&str, Vec)>) { Case::Okay => okay += 1, Case::Failed { line, src, expected, found } => { println!(); - println!(" - Case failed in file {}.rs in line {}.", file, line); + println!(" ❌ Case failed in file {}.rs in line {}.", file, line); println!(" - Source: {:?}", src); println!(" - Expected: {}", expected); println!(" - Found: {}", found); @@ -119,7 +119,11 @@ macro_rules! case { }); (e $src:expr, [$($e:tt)*]) => ({ - let expected = ErrorMap { errors: list!([$($e)*]) }; + let errors = list!([$($e)*]).into_iter() + .map(|s| s.map(|m| m.to_string())) + .collect(); + + let expected = ErrorMap { errors }; let found = parse($src, ParseContext { scope: &scope() }).2; (expected == found, expected, found) }); @@ -204,8 +208,10 @@ function! { } parse(header, body, ctx) { + let cloned = header.clone(); + header.args.clear(); DebugFn { - header: header.clone(), + header: cloned, body: parse!(optional: body, ctx), } }