Refactor parser 🚇

src/func.rs

@@ -1,9 +1,8 @@
 //! Trait and prelude for custom functions.
 
 use crate::Pass;
-use crate::syntax::ParseContext;
-use crate::syntax::func::FuncHeader;
-use crate::syntax::span::Spanned;
+use crate::syntax::ParseState;
+use crate::syntax::func::FuncCall;
 
 /// Types that are useful for creating your own functions.
 pub mod prelude {
@@ -17,7 +16,6 @@ pub mod prelude {
     pub use crate::syntax::span::{Span, Spanned};
 }
 
-
 /// Parse a function from source code.
 pub trait ParseFunc {
     /// A metadata type whose value is passed into the function parser. This
@@ -33,9 +31,8 @@ pub trait ParseFunc {
 
     /// Parse the header and body into this function given a context.
     fn parse(
-        header: FuncHeader,
-        body: Option<Spanned<&str>>,
-        ctx: ParseContext,
+        header: FuncCall,
+        state: &ParseState,
         metadata: Self::Meta,
     ) -> Pass<Self> where Self: Sized;
 }
@@ -53,8 +50,8 @@ pub trait ParseFunc {
 ///         body: Option<SyntaxModel>,
 ///     }
 ///
-///     parse(header, body, ctx, f) {
-///         let body = body!(opt: body, ctx, f);
+///     parse(header, body, state, f) {
+///         let body = body!(opt: body, state, f);
 ///         let hidden = header.args.pos.get::<bool>(&mut f.problems)
 ///             .or_missing(&mut f.problems, header.name.span, "hidden")
 ///             .unwrap_or(false);
@@ -112,7 +109,7 @@ macro_rules! function {
     (@parse($name:ident, $meta:ty) parse(
         $header:ident,
         $body:ident,
-        $ctx:ident,
+        $state:ident,
         $feedback:ident,
         $metadata:ident
     ) $code:block $($r:tt)*) => {
@@ -120,18 +117,18 @@ macro_rules! function {
             type Meta = $meta;
 
             fn parse(
-                #[allow(unused)] mut header: $crate::syntax::func::FuncHeader,
-                #[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>,
-                #[allow(unused)] $ctx: $crate::syntax::ParseContext,
+                #[allow(unused)] mut call: $crate::syntax::func::FuncCall,
+                #[allow(unused)] $state: &$crate::syntax::ParseState,
                 #[allow(unused)] $metadata: Self::Meta,
             ) -> $crate::Pass<Self> where Self: Sized {
                 let mut feedback = $crate::Feedback::new();
-                #[allow(unused)] let $header = &mut header;
+                #[allow(unused)] let $header = &mut call.header;
+                #[allow(unused)] let $body = &mut call.body;
                 #[allow(unused)] let $feedback = &mut feedback;
 
                 let func = $code;
 
-                for arg in header.args.into_iter() {
+                for arg in call.header.args.into_iter() {
                     error!(@feedback, arg.span, "unexpected argument");
                 }
 
@@ -167,21 +164,20 @@ macro_rules! function {
 /// Parse the body of a function.
 ///
 /// - If the function does not expect a body, use `body!(nope: body, feedback)`.
-/// - If the function can have a body, use `body!(opt: body, ctx, feedback,
+/// - If the function can have a body, use `body!(opt: body, state, feedback,
 ///   decos)`.
 ///
 /// # Arguments
 /// - The `$body` should be of type `Option<Spanned<&str>>`.
-/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for
-///   parsing.
+/// - The `$state` is the parse state to use.
 /// - The `$feedback` should be a mutable references to a
 ///   [`Feedback`](crate::Feedback) struct which is filled with the feedback
 ///   information arising from parsing.
 #[macro_export]
 macro_rules! body {
-    (opt: $body:expr, $ctx:expr, $feedback:expr) => ({
+    (opt: $body:expr, $state:expr, $feedback:expr) => ({
         $body.map(|body| {
-            let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
+            let parsed = $crate::syntax::parse(body.v, body.span.start, $state);
             $feedback.extend(parsed.feedback);
             parsed.output
         })

src/layout/model.rs

@@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> {
         }).await;
 
         // Add the errors generated by the model to the error list.
-        self.feedback.extend_offset(model.span.start, layouted.feedback);
+        self.feedback.extend_offset(layouted.feedback, model.span.start);
 
         for command in layouted.output {
             self.execute_command(command, model.span).await;

src/lib.rs

@@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor};
 use crate::problem::Problems;
 use crate::layout::MultiLayout;
 use crate::style::{LayoutStyle, PageStyle, TextStyle};
-use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse};
+use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse};
 use crate::syntax::span::{Position, SpanVec, offset_spans};
 
 
@@ -42,6 +42,8 @@ macro_rules! pub_use_mod {
     };
 }
 
+#[macro_use]
+mod macros;
 #[macro_use]
 pub mod problem;
 pub mod export;
@@ -57,14 +59,13 @@ pub mod syntax;
 /// Transforms source code into typesetted layouts.
 ///
 /// A typesetter can be configured through various methods.
-#[derive(Debug)]
 pub struct Typesetter {
     /// The font loader shared by all typesetting processes.
     loader: GlobalFontLoader,
     /// The base layouting style.
     style: LayoutStyle,
-    /// The standard library scope.
-    scope: Scope,
+    /// The base parser state.
+    parse_state: ParseState,
     /// Whether to render debug boxes.
     debug: bool,
 }
@@ -84,7 +85,7 @@ impl Typesetter {
         Typesetter {
             loader: RefCell::new(FontLoader::new(provider)),
             style: LayoutStyle::default(),
-            scope: Scope::with_std(),
+            parse_state: ParseState { scope: Scope::with_std() },
             debug: false,
         }
     }
@@ -111,7 +112,7 @@ impl Typesetter {
 
     /// Parse source code into a syntax tree.
     pub fn parse(&self, src: &str) -> Pass<SyntaxModel> {
-        parse(Position::ZERO, src, ParseContext { scope: &self.scope })
+        parse(src, Position::ZERO, &self.parse_state)
     }
 
     /// Layout a syntax tree and return the produced layout.
@@ -204,9 +205,9 @@ impl Feedback {
 
     /// Add more feedback whose spans are local and need to be offset by an
     /// `offset` to be correct for this feedbacks context.
-    pub fn extend_offset(&mut self, offset: Position, other: Feedback) {
-        self.problems.extend(offset_spans(offset, other.problems));
-        self.decos.extend(offset_spans(offset, other.decos));
+    pub fn extend_offset(&mut self, other: Feedback, offset: Position) {
+        self.problems.extend(offset_spans(other.problems, offset));
+        self.decos.extend(offset_spans(other.decos, offset));
     }
 }
 

src/library/mod.rs

@@ -53,10 +53,10 @@ function! {
         body: Option<SyntaxModel>,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         header.args.pos.items.clear();
         header.args.key.pairs.clear();
-        ValFunc { body: body!(opt: body, ctx, f) }
+        ValFunc { body: body!(opt: body, state, f) }
     }
 
     layout(self, ctx, f) {

src/library/page.rs

@@ -12,7 +12,7 @@ function! {
         flip: bool,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         body!(nope: body, f);
         PageSizeFunc {
             paper: header.args.pos.get::<Paper>(&mut f.problems),
@@ -50,7 +50,7 @@ function! {
         padding: PaddingMap,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         body!(nope: body, f);
         PageMarginsFunc {
             padding: PaddingMap::parse(&mut f.problems, &mut header.args),

src/library/spacing.rs

@@ -46,9 +46,9 @@ function! {
 
     type Meta = ContentKind;
 
-    parse(header, body, ctx, f, meta) {
+    parse(header, body, state, f, meta) {
         ContentSpacingFunc {
-            body: body!(opt: body, ctx, f),
+            body: body!(opt: body, state, f),
             content: meta,
             spacing: header.args.pos.get::<f64>(&mut f.problems)
                 .map(|num| num as f32)
@@ -84,7 +84,7 @@ function! {
 
     type Meta = Option<SpecificAxis>;
 
-    parse(header, body, ctx, f, meta) {
+    parse(header, body, state, f, meta) {
         body!(nope: body, f);
         SpacingFunc {
             spacing: if let Some(axis) = meta {

src/macros.rs

@@ -0,0 +1,25 @@
+#![allow(unused)]
+
+/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`.
+/// This fits use cases the `?`-operator does not cover, like:
+/// ```
+/// try_or!(result, continue);
+/// ```
+macro_rules! try_or {
+    ($result:expr, $or:expr $(,)?) => {
+        match $result {
+            Ok(v) => v,
+            Err(_) => { $or }
+        }
+    };
+}
+
+/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`.
+macro_rules! try_opt_or {
+    ($option:expr, $or:expr $(,)?) => {
+        match $option {
+            Some(v) => v,
+            None => { $or }
+        }
+    };
+}

src/syntax/func/mod.rs

@@ -9,22 +9,23 @@ pub_use_mod!(maps);
 pub_use_mod!(keys);
 pub_use_mod!(values);
 
+/// An invocation of a function.
+#[derive(Debug, Clone, PartialEq)]
+pub struct FuncCall<'s> {
+    pub header: FuncHeader,
+    /// The body as a raw string containing what's inside of the brackets.
+    pub body: Option<Spanned<&'s str>>,
+}
 
-/// The parsed header of a function.
+/// The parsed header of a function (everything in the first set of brackets).
 #[derive(Debug, Clone, PartialEq)]
 pub struct FuncHeader {
-    /// The function name, that is:
-    /// ```typst
-    /// [box: w=5cm]
-    ///  ^^^
-    /// ```
     pub name: Spanned<Ident>,
-    /// The arguments passed to the function.
     pub args: FuncArgs,
 }
 
 /// The positional and keyword arguments passed to a function.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct FuncArgs {
     /// The positional arguments.
     pub pos: Tuple,

src/syntax/scope.rs

@@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter};
 
 use crate::Pass;
 use crate::func::ParseFunc;
-use super::func::FuncHeader;
-use super::parsing::ParseContext;
-use super::span::Spanned;
+use super::func::FuncCall;
+use super::parsing::ParseState;
 use super::Model;
 
-
 /// A map from identifiers to function parsers.
 pub struct Scope {
     parsers: HashMap<String, Box<Parser>>,
@@ -50,10 +48,8 @@ impl Scope {
     }
 
     /// Return the parser with the given name if there is one.
-    pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
-        self.parsers.get(name)
-            .map(|x| &**x)
-            .ok_or_else(|| &*self.fallback)
+    pub fn get_parser(&self, name: &str) -> Option<&Parser> {
+        self.parsers.get(name).map(AsRef::as_ref)
     }
 
     /// Return the fallback parser.
@@ -72,16 +68,12 @@ impl Debug for Scope {
 
 /// A function which parses the source of a function into a model type which
 /// implements [`Model`].
-type Parser = dyn Fn(
-    FuncHeader,
-    Option<Spanned<&str>>,
-    ParseContext,
-) -> Pass<Box<dyn Model>>;
+type Parser = dyn Fn(FuncCall, &ParseState) -> Pass<Box<dyn Model>>;
 
 fn parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser>
 where F: ParseFunc + Model + 'static {
-    Box::new(move |h, b, c| {
-        F::parse(h, b, c, metadata.clone())
+    Box::new(move |f, s| {
+        F::parse(f, s, metadata.clone())
             .map(|model| Box::new(model) as Box<dyn Model>)
     })
 }

src/syntax/span.rs

@@ -4,14 +4,14 @@ use std::fmt::{self, Debug, Formatter};
 use std::ops::{Add, Sub};
 use serde::Serialize;
 
-/// A vector of spanned things.
+/// A vector of spanned values of type `T`.
 pub type SpanVec<T> = Vec<Spanned<T>>;
 
 /// [Offset](Span::offset) all spans in a vector of spanned things by a start
 /// position.
 pub fn offset_spans<T>(
-    start: Position,
     vec: SpanVec<T>,
+    start: Position,
 ) -> impl Iterator<Item=Spanned<T>> {
     vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
 }
@@ -95,6 +95,11 @@ impl Span {
         Span { start: pos, end: pos }
     }
 
+    /// Expand a span by merging it with another span.
+    pub fn expand(&mut self, other: Span) {
+        *self = Span::merge(*self, other)
+    }
+
     /// Offset a span by a start position.
     ///
     /// This is, for example, used to translate error spans from function local
@@ -126,7 +131,7 @@ impl Position {
     /// The line 0, column 0 position.
     pub const ZERO: Position = Position { line: 0, column: 0 };
 
-    /// Crete a new instance from line and column.
+    /// Create a new position from line and column.
     pub fn new(line: usize, column: usize) -> Position {
         Position { line, column }
     }

src/syntax/test.rs

@@ -63,13 +63,13 @@ function! {
         pub body: Option<SyntaxModel>,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         let cloned = header.clone();
         header.args.pos.items.clear();
         header.args.key.pairs.clear();
         DebugFn {
             header: cloned,
-            body: body!(opt: body, ctx, f),
+            body: body!(opt: body, state, f),
         }
     }
 

src/syntax/tokens.rs

@@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID;
 use crate::size::Size;
 use super::span::{Position, Span, Spanned};
 
-use self::Token::*;
-use self::TokenizationMode::*;
-
+use Token::*;
+use TokenMode::*;
 
 /// A minimal semantic entity of source code.
 #[derive(Debug, Copy, Clone, PartialEq)]
@@ -152,7 +151,7 @@ impl<'s> Token<'s> {
 #[derive(Debug)]
 pub struct Tokens<'s> {
     src: &'s str,
-    mode: TokenizationMode,
+    mode: TokenMode,
     iter: Peekable<Chars<'s>>,
     position: Position,
     index: usize,
@@ -163,20 +162,22 @@ pub struct Tokens<'s> {
 /// backtick tokens.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
 #[allow(missing_docs)]
-pub enum TokenizationMode {
+pub enum TokenMode {
     Header,
     Body,
 }
 
 impl<'s> Tokens<'s> {
-    /// Create a new token iterator with the given mode where the first token
-    /// span starts an the given `start` position.
-    pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
+    /// Create a new token iterator with the given mode.
+    ///
+    /// The first token's span starts an the given `offset` position instead of
+    /// the zero position.
+    pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> {
         Tokens {
             src,
             mode,
             iter: src.chars().peekable(),
-            position: start,
+            position: offset,
             index: 0,
         }
     }
@@ -341,7 +342,7 @@ impl<'s> Tokens<'s> {
         Function { header, body: Some(Spanned { v: body, span }), terminated }
     }
 
-    fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) {
+    fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) {
         let start = self.index();
         let mut terminated = false;
 
@@ -530,7 +531,6 @@ pub fn is_identifier(string: &str) -> bool {
     true
 }
 
-
 #[cfg(test)]
 #[allow(non_snake_case)]
 mod tests {
@@ -557,7 +557,7 @@ mod tests {
     macro_rules! t {
         ($mode:expr, $source:expr => [$($tokens:tt)*]) => {
             let (exp, spans) = span_vec![$($tokens)*];
-            let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>();
+            let found = Tokens::new($source, Position::ZERO, $mode).collect::<Vec<_>>();
             check($source, exp, found, spans);
         }
     }