konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parsing mathematical expressions ➕

Browse Source
This commit is contained in:
Martin Haug 2020-07-18 14:04:58 +02:00
parent 6f1319e91f
commit bb1350cff5
4 changed files with 273 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/syntax/expr.rs
View File

@ -1,6 +1,6 @@
//! Expressions in function headers.
use std::fmt::{self, Write, Debug, Formatter};
use std::fmt::{self, Debug, Formatter};
use std::iter::FromIterator;
use std::ops::Deref;
use std::str::FromStr;
@ -26,7 +26,7 @@ pub enum Expr {
Size(Size),
/// A bool: `true, false`.
Bool(bool),
/// A color value, including the alpha channel: `#f79143ff`
/// A color value, including the alpha channel: `#f79143ff`.
Color(RgbaColor),
/// A tuple: `(false, 12cm, "hi")`.
Tuple(Tuple),
@ -34,6 +34,16 @@ pub enum Expr {
NamedTuple(NamedTuple),
/// An object: `{ fit: false, size: 12pt }`.
Object(Object),
/// An operator that negates the contained expression.
Neg(Box<Spanned<Expr>>),
/// An operator that adds the contained expressions.
Add(Box<Spanned<Expr>>, Box<Spanned<Expr>>),
/// An operator that subtracts contained expressions.
Sub(Box<Spanned<Expr>>, Box<Spanned<Expr>>),
/// An operator that multiplies the contained expressions.
Mul(Box<Spanned<Expr>>, Box<Spanned<Expr>>),
/// An operator that divides the contained expressions.
Div(Box<Spanned<Expr>>, Box<Spanned<Expr>>),
}
impl Expr {
@ -50,6 +60,11 @@ impl Expr {
Tuple(_) => "tuple",
NamedTuple(_) => "named tuple",
Object(_) => "object",
Neg(_) => "negation",
Add(_, _) => "addition",
Sub(_, _) => "subtraction",
Mul(_, _) => "multiplication",
Div(_, _) => "division",
}
}
}
@ -67,6 +82,11 @@ impl Debug for Expr {
Tuple(t) => t.fmt(f),
NamedTuple(t) => t.fmt(f),
Object(o) => o.fmt(f),
Neg(e) => write!(f, "-{:?}", e),
Add(a, b) => write!(f, "({:?} + {:?})", a, b),
Sub(a, b) => write!(f, "({:?} - {:?})", a, b),
Mul(a, b) => write!(f, "({:?} * {:?})", a, b),
Div(a, b) => write!(f, "({:?} / {:?})", a, b),
}
}
}
@ -102,9 +122,7 @@ impl Ident {
impl Debug for Ident {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
f.write_char('`')?;
f.write_str(&self.0)?;
f.write_char('`')
write!(f, "`{}`", self.0)
}
}

203
src/syntax/parsing.rs
View File

@ -223,9 +223,109 @@ impl<'s> FuncParser<'s> {
}
}).v
}
/// Parse an atomic or compound (tuple / object) expression.
/// Parse an expression which may contain math operands.
/// For this, this method looks for operators in
/// descending order of associativity, i.e. we first drill down to find all
/// negations, brackets and tuples, the next level,
/// we look for multiplication and division and here finally, for addition
/// and subtraction.
fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
let term = self.parse_term()?;
self.skip_whitespace();
if let Some(next) = self.peek() {
match next.v {
Token::Plus | Token::Hyphen => {
self.eat();
self.skip_whitespace();
let o2 = self.parse_expr();
if o2.is_none() {
self.feedback.errors.push(err!(
Span::merge(next.span, term.span);
"Missing right summand"
));
return Some(term)
}
let o2 = o2.expect("Checked for None before");
let span = Span::merge(term.span, o2.span);
match next.v {
Token::Plus => Some(Spanned::new(
Expr::Add(Box::new(term), Box::new(o2)), span
)),
Token::Hyphen => Some(Spanned::new(
Expr::Sub(Box::new(term), Box::new(o2)), span
)),
_ => unreachable!(),
}
},
_ => Some(term)
}
} else {
Some(term)
}
}
fn parse_term(&mut self) -> Option<Spanned<Expr>> {
// TODO: Deduplicate code here
let factor = self.parse_factor()?;
self.skip_whitespace();
if let Some(next) = self.peek() {
match next.v {
Token::Star | Token::Slash => {
self.eat();
self.skip_whitespace();
let o2 = self.parse_term();
if o2.is_none() {
self.feedback.errors.push(err!(
Span::merge(next.span, factor.span);
"Missing right factor"
));
return Some(factor)
}
let o2 = o2.expect("Checked for None before");
let span = Span::merge(factor.span, o2.span);
match next.v {
Token::Star => Some(Spanned::new(
Expr::Mul(Box::new(factor), Box::new(o2)), span
)),
Token::Slash => Some(Spanned::new(
Expr::Div(Box::new(factor), Box::new(o2)), span
)),
_ => unreachable!(),
}
},
_ => Some(factor)
}
} else {
Some(factor)
}
}
/// Parse expressions that are of the form value or -value
fn parse_factor(&mut self) -> Option<Spanned<Expr>> {
let first = self.peek()?;
if first.v == Token::Hyphen {
self.eat();
let o2 = self.parse_value();
self.skip_whitespace();
if o2.is_none() {
self.feedback.errors.push(err!(first.span; "Dangling minus"));
return None
}
let o2 = o2.expect("Checked for None before");
let span = Span::merge(first.span, o2.span);
Some(Spanned::new(Expr::Neg(Box::new(o2)), span))
} else {
self.parse_value()
}
}
fn parse_value(&mut self) -> Option<Spanned<Expr>> {
let first = self.peek()?;
macro_rules! take {
($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } });
@ -263,27 +363,36 @@ impl<'s> FuncParser<'s> {
}
},
Token::LeftParen => self.parse_tuple().map(|t| Expr::Tuple(t)),
Token::LeftParen => {
let mut tuple = self.parse_tuple();
// Coerce 1-tuple into value
if tuple.1 && tuple.0.v.items.len() > 0 {
tuple.0.v.items.pop().expect("Length is one")
} else {
tuple.0.map(|t| Expr::Tuple(t))
}
},
Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)),
_ => return None,
})
}
/// Parse a tuple expression: `(<expr>, ...)`.
fn parse_tuple(&mut self) -> Spanned<Tuple> {
/// Parse a tuple expression: `(<expr>, ...)`. The boolean in the return
/// values showes whether the tuple can be coerced into a single value
fn parse_tuple(&mut self) -> (Spanned<Tuple>, bool) {
let token = self.eat();
debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen));
// Parse a collection until a right paren appears and complain about
// missing a `value` when an invalid token is encoutered.
self.parse_collection(Some(Token::RightParen),
self.parse_collection_bracket_aware(Some(Token::RightParen),
|p| p.parse_expr().ok_or(("value", None)))
}
/// Parse a tuple expression: `name(<expr>, ...)` with a given identifier.
fn parse_named_tuple(&mut self, name: Spanned<Ident>) -> Spanned<NamedTuple> {
let tuple = self.parse_tuple();
let tuple = self.parse_tuple().0;
let span = Span::merge(name.span, tuple.span);
Spanned::new(NamedTuple::new(name, tuple), span)
}
@ -319,17 +428,21 @@ impl<'s> FuncParser<'s> {
}
/// Parse a comma-separated collection where each item is parsed through
/// `parse_item` until the `end` token is met.
fn parse_collection<C, I, F>(
/// `parse_item` until the `end` token is met. The first item in the return
/// tuple is the collection, the second item indicates whether the
/// collection can be coerced into a single item
/// (i.e. at least one comma appeared).
fn parse_collection_bracket_aware<C, I, F>(
&mut self,
end: Option<Token>,
mut parse_item: F
) -> Spanned<C>
) -> (Spanned<C>, bool)
where
C: FromIterator<I>,
F: FnMut(&mut Self) -> Result<I, (&'static str, Option<Position>)>,
{
let start = self.pos();
let mut can_be_coerced = true;
// Parse the comma separated items.
let collection = std::iter::from_fn(|| {
@ -359,8 +472,14 @@ impl<'s> FuncParser<'s> {
let behind_item = self.pos();
self.skip_whitespace();
match self.peekv() {
Some(Token::Comma) => { self.eat(); }
t @ Some(_) if t != end => self.expected_at("comma", behind_item),
Some(Token::Comma) => {
can_be_coerced = false;
self.eat();
}
t @ Some(_) if t != end => {
can_be_coerced = false;
self.expected_at("comma", behind_item);
},
_ => {}
}
@ -383,7 +502,21 @@ impl<'s> FuncParser<'s> {
}).filter_map(|x| x).collect();
let end = self.pos();
Spanned::new(collection, Span { start, end })
(Spanned::new(collection, Span { start, end }), can_be_coerced)
}
/// Parse a comma-separated collection where each item is parsed through
/// `parse_item` until the `end` token is met.
fn parse_collection<C, I, F>(
&mut self,
end: Option<Token>,
parse_item: F
) -> Spanned<C>
where
C: FromIterator<I>,
F: FnMut(&mut Self) -> Result<I, (&'static str, Option<Position>)>,
{
self.parse_collection_bracket_aware(end, parse_item).0
}
/// Try to parse an identifier and do nothing if the peekable token is no
@ -546,6 +679,19 @@ mod tests {
fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) }
fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) }
fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) }
fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) }
fn Add(e1: Expr, e2: Expr) -> Expr {
Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2)))
}
fn Sub(e1: Expr, e2: Expr) -> Expr {
Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2)))
}
fn Mul(e1: Expr, e2: Expr) -> Expr {
Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2)))
}
fn Div(e1: Expr, e2: Expr) -> Expr {
Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2)))
}
fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr {
Expr::Color(RgbaColor::new(r, g, b, a))
@ -814,6 +960,12 @@ mod tests {
p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]);
p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]);
// Math
p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]);
p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]);
p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]);
p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]);
// Unclosed string.
p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [
(0:13, 0:13, "expected quote"),
@ -835,6 +987,19 @@ mod tests {
]);
}
#[test]
fn parse_complex_mathematical_expressions() {
p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": (
Mul(
Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))),
Sub(Div(Num(5.0), Num(2.0)), Num(1.0))
)
), {})]);
p!("[val: (6.3E+2+4*-3.2pt)/2]" => [func!("val": (
Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0))
), {})]);
}
#[test]
fn parse_tuples() {
// Empty tuple
@ -858,9 +1023,9 @@ mod tests {
);
// Unclosed tuple
p!("[val: (hello]" =>
[func!("val": (tuple!(Id("hello"))), {})],
[(0:12, 0:12, "expected closing paren")],
p!("[val: (hello,]" =>
[func!("val": (tuple!(Id("hello"),)), {})],
[(0:13, 0:13, "expected closing paren")],
);
p!("[val: lang(中文]" =>
[func!("val": (named_tuple!("lang", Id("中文"))), {})],
@ -882,8 +1047,8 @@ mod tests {
);
// Nested tuples
p!("[val: (1, (2))]" =>
[func!("val": (tuple!(Num(1.0), tuple!(Num(2.0)))), {})]
p!("[val: (1, (2, 3))]" =>
[func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})]
);
p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" =>
[func!("val": (named_tuple!(
@ -1085,7 +1250,7 @@ mod tests {
// Body nodes in bodies.
p!("[val:*][*Hi*]" =>
[func!("val"; [Bold, T("Hi"), Bold])],
[(0:5, 0:6, "expected argument, found invalid token")],
[(0:5, 0:6, "expected argument, found star")],
);
// Errors in bodies.

31
src/syntax/test.rs
View File

@ -88,13 +88,6 @@ pub trait SpanlessEq<Rhs=Self> {
fn spanless_eq(&self, other: &Rhs) -> bool;
}
impl<T: SpanlessEq> SpanlessEq for Vec<Spanned<T>> {
fn spanless_eq(&self, other: &Vec<Spanned<T>>) -> bool {
self.len() == other.len()
&& self.iter().zip(other).all(|(x, y)| x.v.spanless_eq(&y.v))
}
}
impl SpanlessEq for SyntaxModel {
fn spanless_eq(&self, other: &SyntaxModel) -> bool {
self.nodes.spanless_eq(&other.nodes)
@ -130,6 +123,11 @@ impl SpanlessEq for Expr {
(Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
(Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
(Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
(Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b),
(Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
(Expr::Sub(a1, a2), Expr::Sub(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
(Expr::Mul(a1, a2), Expr::Mul(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
(Expr::Div(a1, a2), Expr::Div(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
(a, b) => a == b,
}
}
@ -158,6 +156,25 @@ impl SpanlessEq for Object {
}
}
impl<T: SpanlessEq> SpanlessEq for Vec<T> {
fn spanless_eq(&self, other: &Vec<T>) -> bool {
self.len() == other.len()
&& self.iter().zip(other).all(|(x, y)| x.spanless_eq(&y))
}
}
impl<T: SpanlessEq> SpanlessEq for Spanned<T> {
fn spanless_eq(&self, other: &Spanned<T>) -> bool {
self.v.spanless_eq(&other.v)
}
}
impl<T: SpanlessEq> SpanlessEq for Box<T> {
fn spanless_eq(&self, other: &Box<T>) -> bool {
(&**self).spanless_eq(&**other)
}
}
/// Implement `SpanlessEq` by just forwarding to `PartialEq`.
macro_rules! forward {
($type:ty) => {

51
src/syntax/tokens.rs
View File

@ -78,10 +78,18 @@ pub enum Token<'s> {
ExprSize(Size),
/// A boolean in a function header: `true | false`.
ExprBool(bool),
/// A hex value in a function header: `#20d82a`
/// A hex value in a function header: `#20d82a`.
ExprHex(&'s str),
/// A plus in a function header, signifying the addition of expressions.
Plus,
/// A hyphen in a function header,
/// signifying the subtraction of expressions.
Hyphen,
/// A slash in a function header, signifying the division of expressions.
Slash,
/// A star in body-text.
/// A star. It can appear in a function header where it signifies the
/// multiplication of expressions or the body where it modifies the styling.
Star,
/// An underscore in body-text.
Underscore,
@ -125,6 +133,9 @@ impl<'s> Token<'s> {
ExprSize(_) => "size",
ExprBool(_) => "bool",
ExprHex(_) => "hex value",
Plus => "plus",
Hyphen => "minus",
Slash => "slash",
Star => "star",
Underscore => "underscore",
Backslash => "backslash",
@ -212,12 +223,20 @@ impl<'s> Iterator for Tokens<'s> {
':' if self.mode == Header => Colon,
',' if self.mode == Header => Comma,
'=' if self.mode == Header => Equals,
// Expression operators.
'+' if self.mode == Header => Plus,
'-' if self.mode == Header => Hyphen,
'/' if self.mode == Header => Slash,
// String values.
'"' if self.mode == Header => self.parse_string(),
// Star serves a double purpose as a style modifier
// and a expression operator in the header.
'*' => Star,
// Style toggles.
'*' if self.mode == Body => Star,
'_' if self.mode == Body => Underscore,
'`' if self.mode == Body => self.parse_raw(),
@ -231,15 +250,20 @@ impl<'s> Iterator for Tokens<'s> {
c => {
let body = self.mode == Body;
let mut last_was_e = false;
let text = self.read_string_until(|n| {
match n {
let val = match n {
c if c.is_whitespace() => true,
'[' | ']' | '/' => true,
'\\' | '*' | '_' | '`' if body => true,
'[' | ']' | '/' | '*' => true,
'\\' | '_' | '`' if body => true,
':' | '=' | ',' | '"' |
'(' | ')' | '{' | '}' if !body => true,
'+' | '-' if !body && !last_was_e => true,
_ => false,
}
};
last_was_e = n == 'e' || n == 'E';
val
}, false, -(c.len_utf8() as isize), 0).0;
if self.mode == Header {
@ -411,6 +435,8 @@ impl<'s> Tokens<'s> {
}
}
/// Will read the input stream until the argument F evaluates to `true`
/// for the current character.
fn read_string_until<F>(
&mut self,
mut f: F,
@ -517,6 +543,10 @@ mod tests {
ExprBool as Bool,
ExprHex as Hex,
Text as T,
Plus,
Hyphen as Min,
Star,
Slash,
};
#[allow(non_snake_case)]
@ -595,7 +625,7 @@ mod tests {
t!(Body, "`]" => [Raw("]", false)]);
t!(Body, "`\\``" => [Raw("\\`", true)]);
t!(Body, "\\ " => [Backslash, S(0)]);
t!(Header, "_*`" => [Invalid("_*`")]);
t!(Header, "_`" => [Invalid("_`")]);
}
#[test]
@ -613,10 +643,13 @@ mod tests {
t!(Header, "12e4%" => [Num(1200.0)]);
t!(Header, "__main__" => [Id("__main__")]);
t!(Header, ".func.box" => [Id(".func.box")]);
t!(Header, "--arg, _b, _1" => [Id("--arg"), Comma, S(0), Id("_b"), Comma, S(0), Id("_1")]);
t!(Header, "arg, _b, _1" => [Id("arg"), Comma, S(0), Id("_b"), Comma, S(0), Id("_1")]);
t!(Header, "12_pt, 12pt" => [Invalid("12_pt"), Comma, S(0), Sz(Size::pt(12.0))]);
t!(Header, "1e5in" => [Sz(Size::inches(100000.0))]);
t!(Header, "2.3cm" => [Sz(Size::cm(2.3))]);
t!(Header, "12e-3in" => [Sz(Size::inches(12e-3))]);
t!(Header, "6.1cm + 4pt,a=1*2" => [Sz(Size::cm(6.1)), S(0), Plus, S(0), Sz(Size::pt(4.0)), Comma, Id("a"), Equals, Num(1.0), Star, Num(2.0)]);
t!(Header, "(5 - 1) / 2.1" => [LP, Num(5.0), S(0), Min, S(0), Num(1.0), RP, S(0), Slash, S(0), Num(2.1)]);
t!(Header, "02.4mm" => [Sz(Size::mm(2.4))]);
t!(Header, "2.4.cm" => [Invalid("2.4.cm")]);
t!(Header, "(1,2)" => [LP, Num(1.0), Comma, Num(2.0), RP]);