konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do argument parsing ☑

Browse Source
This commit is contained in:
Laurenz 2020-01-16 23:00:39 +01:00
parent 08b91a265f
commit 70878885f5
5 changed files with 332 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/macros.rs
View File

@ -62,3 +62,13 @@ macro_rules! pub_use_mod {
pub use $name::*;
};
}
/// Whether an expression matches a set of patterns.
macro_rules! matches {
($expression:expr, $( $pattern:pat )|+ $( if $guard: expr )?) => {
match $expression {
$( $pattern )|+ $( if $guard )? => true,
_ => false
}
}
}

50
src/syntax/mod.rs
View File

@ -154,6 +154,22 @@ pub struct FuncArgs {
pub keyword: Object,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Arg {
Pos(Spanned<Expression>),
Key(Pair),
}
impl Arg {
/// The span or the value or combined span of key and value.
pub fn span(&self) -> Span {
match self {
Arg::Pos(spanned) => spanned.span,
Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span),
}
}
}
impl FuncArgs {
pub fn new() -> FuncArgs {
FuncArgs {
@ -246,32 +262,40 @@ pub struct Colorization {
pub tokens: Vec<Spanned<ColorToken>>,
}
impl Colorization {
pub fn new() -> Colorization {
Colorization { tokens: vec![] }
}
pub fn add(&mut self, token: ColorToken, span: Span) {
self.tokens.push(Spanned { v: token, span });
}
pub fn replace_last(&mut self, token: ColorToken) {
self.tokens.last_mut().expect("replace_last: no token").v = token;
}
}
/// Entities which can be colored by syntax highlighting.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum ColorToken {
Comment,
Bracket,
FuncName,
Colon,
Key,
Equals,
Comma,
Paren,
Brace,
ExprIdent,
ExprStr,
ExprNumber,
ExprSize,
ExprBool,
Bold,
Italic,
Monospace,
Invalid,
}
@ -279,3 +303,17 @@ pub enum ColorToken {
pub struct ErrorMap {
pub errors: Vec<Spanned<String>>,
}
impl ErrorMap {
pub fn new() -> ErrorMap {
ErrorMap { errors: vec![] }
}
pub fn add(&mut self, message: impl Into<String>, span: Span) {
self.errors.push(Spanned { v: message.into(), span });
}
pub fn add_at(&mut self, message: impl Into<String>, pos: Position) {
self.errors.push(Spanned { v: message.into(), span: Span::at(pos) })
}
}

372
src/syntax/parsing.rs
View File

@ -20,7 +20,6 @@ struct Parser<'s> {
ctx: ParseContext<'s>,
colorization: Colorization,
error_map: ErrorMap,
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
position: Position,
@ -32,9 +31,8 @@ impl<'s> Parser<'s> {
Parser {
src,
ctx,
error_map: ErrorMap { errors: vec![] },
colorization: Colorization { tokens: vec![] },
error_map: ErrorMap::new(),
colorization: Colorization::new(),
tokens: Tokens::new(src),
peeked: None,
position: Position::ZERO,
@ -42,6 +40,7 @@ impl<'s> Parser<'s> {
}
}
/// The main parsing entrypoint.
fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) {
let mut tree = SyntaxTree::new();
@ -79,10 +78,18 @@ impl<'s> Parser<'s> {
(tree, self.colorization, self.error_map)
}
/// Parses a function including header and body with the cursor starting
/// right behind the first opening bracket.
fn parse_func(&mut self) -> Option<Spanned<Node>> {
let start = self.last_pos();
let header = self.parse_func_header();
self.eat_until(|t| t == RightBracket, false);
if self.eat().map(Spanned::value) != Some(RightBracket) {
self.expected_at("closing bracket", self.pos());
}
let call = self.parse_func_call(header)?;
let end = self.pos();
@ -91,21 +98,17 @@ impl<'s> Parser<'s> {
Some(Spanned { v: Node::Func(call), span })
}
/// Parses a function header including the closing bracket.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
self.skip_whitespace();
let name = self.parse_func_name().or_else(|| {
self.eat_until(|t| t == RightBracket, true);
None
})?;
let name = self.parse_func_name()?;
self.skip_whitespace();
let args = match self.eat() {
Some(Spanned { v: Colon, .. }) => self.parse_func_args(),
Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
other => {
self.expected("colon or closing bracket", other);
self.eat_until(|t| t == RightBracket, true);
self.expected_at("colon or closing bracket", name.span.end);
FuncArgs::new()
}
};
@ -113,8 +116,140 @@ impl<'s> Parser<'s> {
Some(FuncHeader { name, args })
}
/// Parses the function name if is the next token. Otherwise, it adds an
/// error and returns `None`.
fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
match self.eat() {
Some(Spanned { v: ExprIdent(ident), span }) => {
self.colorization.replace_last(ColorToken::FuncName);
return Some(Spanned { v: Ident(ident.to_string()), span });
}
other => self.expected_found_or_at("identifier", other, self.pos()),
}
None
}
/// Parses the function arguments and stops right before the final closing
/// bracket.
fn parse_func_args(&mut self) -> FuncArgs {
let mut args = FuncArgs::new();
loop {
self.skip_whitespace();
match self.peekv() {
Some(RightBracket) | None => break,
_ => match self.parse_arg() {
Some(Arg::Pos(item)) => args.add_pos(item),
Some(Arg::Key(pair)) => args.add_key_pair(pair),
None => {}
}
}
}
args
}
/// Parse a positional or keyword argument.
fn parse_arg(&mut self) -> Option<Arg> {
let first = self.peek()?;
let span = first.span;
let arg = if let ExprIdent(ident) = first.v {
self.eat();
self.skip_whitespace();
let ident = Ident(ident.to_string());
if let Some(Equals) = self.peekv() {
self.colorization.replace_last(ColorToken::Key);
self.eat();
self.skip_whitespace();
self.parse_expr().map(|value| {
Arg::Key(Pair {
key: Spanned { v: ident, span },
value,
})
})
} else {
Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
}
} else {
self.parse_expr().map(|expr| Arg::Pos(expr))
};
if let Some(arg) = &arg {
self.skip_whitespace();
match self.peekv() {
Some(RightBracket) => {}
Some(Comma) => { self.eat(); }
Some(_) => self.expected_at("comma", arg.span().end),
_ => {}
}
} else {
let found = self.eat();
self.expected_found_or_at("value", found, self.pos());
}
arg
}
/// Parse a atomic or compound (tuple / object) expression.
fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
let first = self.peek()?;
let mut expr = |v| {
self.eat();
Spanned { v, span: first.span }
};
Some(match first.v {
ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
ExprStr(s) => expr(Expression::Str(s.to_string())),
ExprNumber(n) => expr(Expression::Number(n)),
ExprSize(s) => expr(Expression::Size(s)),
ExprBool(b) => expr(Expression::Bool(b)),
LeftParen => self.parse_tuple(),
LeftBrace => self.parse_object(),
_ => return None,
})
}
/// Parse a tuple expression.
fn parse_tuple(&mut self) -> Spanned<Expression> {
let start = self.pos();
// TODO: Do the thing.
self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
if self.peekv() == Some(RightParen) {
self.eat();
}
let end = self.pos();
let span = Span { start, end };
Spanned { v: Expression::Tuple(Tuple::new()), span }
}
/// Parse an object expression.
fn parse_object(&mut self) -> Spanned<Expression> {
let start = self.pos();
// TODO: Do the thing.
self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
if self.peekv() == Some(RightBrace) {
self.eat();
}
let end = self.pos();
let span = Span { start, end };
Spanned { v: Expression::Object(Object::new()), span }
}
/// Parse the body of a function invocation.
fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> {
let body = if self.peek() == Some(LeftBracket) {
let body = if self.peekv() == Some(LeftBracket) {
self.eat();
let start = self.tokens.index();
@ -127,9 +262,10 @@ impl<'s> Parser<'s> {
let body = &self.src[start .. end];
if found {
assert_eq!(self.eat().map(Spanned::value), Some(RightBracket));
let next = self.eat().map(Spanned::value);
debug_assert_eq!(next, Some(RightBracket));
} else {
self.error_here("expected closing bracket");
self.expected_at("closing bracket", self.pos());
}
Some(body)
@ -139,111 +275,57 @@ impl<'s> Parser<'s> {
let header = header?;
let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| {
self.error(
format!("unknown function: `{}`", header.name.v),
header.name.span
);
let message = format!("unknown function: `{}`", header.name.v);
self.error_map.add(message, header.name.span);
None
})?;
Some(FuncCall(parser(header, body, self.ctx).unwrap()))
}
fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
match self.eat() {
Some(Spanned { v: ExprIdent(ident), span }) => {
self.color(Spanned { v: ColorToken::FuncName, span }, true);
Some(Spanned { v: Ident(ident.to_string()), span })
}
other => {
self.expected("identifier", other);
None
}
}
}
fn parse_func_args(&mut self) -> FuncArgs {
// todo!()
self.eat_until(|t| t == RightBracket, true);
FuncArgs::new()
}
fn parse_tuple(&mut self) -> Spanned<Expression> {
todo!("parse_tuple")
}
fn parse_object(&mut self) -> Spanned<Expression> {
todo!("parse_object")
}
/// Skip all whitespace/comment tokens.
fn skip_whitespace(&mut self) {
self.eat_until(|t| match t {
Whitespace(_) | LineComment(_) | BlockComment(_) => false,
_ => true,
}, false)
self.eat_until(|t|
!matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
}
fn expected(&mut self, thing: &str, found: Option<Spanned<Token>>) {
if let Some(Spanned { v: found, span }) = found {
self.error(
format!("expected {}, found {}", thing, name(found)),
span
);
} else {
self.error_here(format!("expected {}", thing));
/// Add an error about an `thing` which was expected but not found at the
/// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
self.error_map.add_at(format!("expected {}", thing), pos);
}
/// Add an error about an expected `thing` which was not found, showing
/// what was found instead.
fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
let message = format!("expected {}, found {}", thing, name(found.v));
self.error_map.add(message, found.span);
}
/// Add a found-error if `found` is some and a positional error, otherwise.
fn expected_found_or_at(
&mut self,
thing: &str,
found: Option<Spanned<Token>>,
pos: Position
) {
match found {
Some(found) => self.expected_found(thing, found),
None => self.expected_at(thing, pos),
}
}
/// Add an error about an unexpected token `found`.
fn unexpected(&mut self, found: Spanned<Token>) {
self.error_map.errors.push(found.map(|t| format!("unexpected {}", name(t))));
}
fn error(&mut self, message: impl Into<String>, span: Span) {
self.error_map.errors.push(Spanned { v: message.into(), span });
}
fn error_here(&mut self, message: impl Into<String>) {
self.error(message, Span::at(self.pos()));
}
fn color(&mut self, token: Spanned<ColorToken>, replace_last: bool) {
if replace_last {
if let Some(last) = self.colorization.tokens.last_mut() {
*last = token;
return;
}
}
self.colorization.tokens.push(token);
}
fn color_token(&mut self, token: Spanned<Token<'s>>) {
let colored = match token.v {
LineComment(_) | BlockComment(_) => Some(ColorToken::Comment),
StarSlash => Some(ColorToken::Invalid),
LeftBracket | RightBracket => Some(ColorToken::Bracket),
LeftParen | RightParen => Some(ColorToken::Paren),
LeftBrace | RightBrace => Some(ColorToken::Brace),
Colon => Some(ColorToken::Colon),
Comma => Some(ColorToken::Comma),
Equals => Some(ColorToken::Equals),
ExprIdent(_) => Some(ColorToken::ExprIdent),
ExprStr(_) => Some(ColorToken::ExprStr),
ExprNumber(_) => Some(ColorToken::ExprNumber),
ExprSize(_) => Some(ColorToken::ExprSize),
ExprBool(_) => Some(ColorToken::ExprBool),
_ => None,
};
if let Some(color) = colored {
self.colorization.tokens.push(Spanned { v: color, span: token.span });
}
self.error_map.add(format!("unexpected {}", name(found.v)), found.span);
}
/// Consume tokens until the function returns true and only consume the last
/// token if instructed to.
fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
where F: FnMut(Token<'s>) -> bool {
while let Some(token) = self.peek() {
if f(token) {
if f(token.v) {
if eat_match {
self.eat();
}
@ -254,56 +336,90 @@ impl<'s> Parser<'s> {
}
}
/// Consume and return the next token, update positions and colorize the
/// token. All colorable tokens are per default colorized here, to override
/// a colorization use `Colorization::replace_last`.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
let token = self.peeked.take().unwrap_or_else(|| self.tokens.next());
let token = self.peeked.take()
.unwrap_or_else(|| self.tokens.next());
self.last_position = self.position;
if let Some(spanned) = token {
self.color_token(spanned);
self.position = spanned.span.end;
if let Some(token) = token {
if let Some(color) = color(token.v) {
self.colorization.add(color, token.span);
}
self.last_position = self.position;
self.position = token.span.end;
}
token
}
fn peek(&mut self) -> Option<Token<'s>> {
/// Peek at the next token without consuming it.
fn peek(&mut self) -> Option<Spanned<Token<'s>>> {
let iter = &mut self.tokens;
self.peeked
.get_or_insert_with(|| iter.next())
.map(Spanned::value)
*self.peeked.get_or_insert_with(|| iter.next())
}
fn peekv(&mut self) -> Option<Token<'s>> {
self.peek().map(Spanned::value)
}
/// The position at the end of the last eat token / start of the peekable
/// token.
fn pos(&self) -> Position {
self.position
}
/// The position at the start of the last eaten token.
fn last_pos(&self) -> Position {
self.last_position
}
}
/// The name of a token in an `expected <...>` error.
fn name(token: Token) -> &'static str {
match token {
Whitespace(_) => "whitespace",
LineComment(_) | BlockComment(_) => "comment",
StarSlash => "end of block comment",
LeftBracket => "opening bracket",
RightBracket => "closing bracket",
LeftParen => "opening paren",
RightParen => "closing paren",
LeftBrace => "opening brace",
RightBrace => "closing brace",
Colon => "colon",
Comma => "comma",
Equals => "equals sign",
ExprIdent(_) => "identifier",
ExprStr(_) => "string",
StarSlash => "end of block comment",
LeftBracket => "opening bracket",
RightBracket => "closing bracket",
LeftParen => "opening paren",
RightParen => "closing paren",
LeftBrace => "opening brace",
RightBrace => "closing brace",
Colon => "colon",
Comma => "comma",
Equals => "equals sign",
ExprIdent(_) => "identifier",
ExprStr(_) => "string",
ExprNumber(_) => "number",
ExprSize(_) => "size",
ExprBool(_) => "bool",
Star => "star",
Underscore => "underscore",
Backtick => "backtick",
Text(_) => "text",
ExprSize(_) => "size",
ExprBool(_) => "bool",
Star => "star",
Underscore => "underscore",
Backtick => "backtick",
Text(_) => "invalid identifier",
}
}
/// The color token corresponding to a token.
fn color(token: Token) -> Option<ColorToken> {
Some(match token {
LineComment(_) | BlockComment(_) => ColorToken::Comment,
LeftBracket | RightBracket => ColorToken::Bracket,
LeftParen | RightParen => ColorToken::Paren,
LeftBrace | RightBrace => ColorToken::Brace,
Colon => ColorToken::Colon,
Comma => ColorToken::Comma,
Equals => ColorToken::Equals,
ExprIdent(_) => ColorToken::ExprIdent,
ExprStr(_) => ColorToken::ExprStr,
ExprNumber(_) => ColorToken::ExprNumber,
ExprSize(_) => ColorToken::ExprSize,
ExprBool(_) => ColorToken::ExprBool,
StarSlash => ColorToken::Invalid,
_ => return None,
})
}

37
tests/parser/trees.rs
View File

@ -1,15 +1,19 @@
// Basics.
p "" => []
p "hi" => [T("hi")]
p "hi you" => [T("hi"), S, T("you")]
p "❤\n\n 🌍" => [T("❤"), N, T("🌍")]
// Functions.
p "[func]" => [func!("func"; None)]
p "[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, B, T("you"), B]))]
p "from [align: left] to" => [
T("from"), S, func!("align", pos: [ID("left")]; None), S, T("to"),
]
p "[f: left, 12pt, false]" => [
func!("f", pos: [ID("left"), SIZE(Size::pt(12.0)), BOOL(false)]; None)
]
p "[f: , hi, * \"du\"]" => [func!("f", pos: [ID("hi"), STR("du")]; None)]
p "[box: x=1.2pt, false][a b c] bye" => [
func!(
"box",
@ -20,14 +24,23 @@ p "[box: x=1.2pt, false][a b c] bye" => [
S, T("bye"),
]
c "hi" => []
c "[align: left][\n _body_\n]" => [
(0:0, 0:1, B),
(0:1, 0:6, FN),
(0:6, 0:7, CL),
(0:8, 0:12, ID),
(0:12, 0:13, B),
(0:13, 0:14, B),
(1:4, 1:10, IT),
(2:0, 2:2, B),
// Errors.
e "[f: , hi, * \"du\"]" => [
(0:4, 0:5, "expected value, found comma"),
(0:10, 0:11, "expected value, found invalid identifier"),
]
e "[f:, , ,]" => [
(0:3, 0:4, "expected value, found comma"),
(0:5, 0:6, "expected value, found comma"),
(0:7, 0:8, "expected value, found comma"),
]
e "[f:" => [(0:3, 0:3, "expected closing bracket")]
e "[f: hi" => [(0:6, 0:6, "expected closing bracket")]
e "[f: hey 12pt]" => [(0:7, 0:7, "expected comma")]
e "[box: x=, false y=z=4" => [
(0:8, 0:9, "expected value, found comma"),
(0:15, 0:15, "expected comma"),
(0:19, 0:19, "expected comma"),
(0:19, 0:20, "expected value, found equals sign"),
(0:21, 0:21, "expected closing bracket"),
]

12
tests/src/parser.rs
View File

@ -40,7 +40,7 @@ fn test(tests: Vec<(&str, Vec<Case>)>) {
Case::Okay => okay += 1,
Case::Failed { line, src, expected, found } => {
println!();
println!(" - Case failed in file {}.rs in line {}.", file, line);
println!(" ❌ Case failed in file {}.rs in line {}.", file, line);
println!(" - Source: {:?}", src);
println!(" - Expected: {}", expected);
println!(" - Found: {}", found);
@ -119,7 +119,11 @@ macro_rules! case {
});
(e $src:expr, [$($e:tt)*]) => ({
let expected = ErrorMap { errors: list!([$($e)*]) };
let errors = list!([$($e)*]).into_iter()
.map(|s| s.map(|m| m.to_string()))
.collect();
let expected = ErrorMap { errors };
let found = parse($src, ParseContext { scope: &scope() }).2;
(expected == found, expected, found)
});
@ -204,8 +208,10 @@ function! {
}
parse(header, body, ctx) {
let cloned = header.clone();
header.args.clear();
DebugFn {
header: header.clone(),
header: cloned,
body: parse!(optional: body, ctx),
}
}