diff --git a/.gitignore b/.gitignore index cf34a4599..1c9ae1a2d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ **/*.rs.bk Cargo.lock tests/cache +_things diff --git a/src/bin/main.rs b/src/bin/main.rs index 231ac4875..8cb7c0554 100644 --- a/src/bin/main.rs +++ b/src/bin/main.rs @@ -41,7 +41,7 @@ fn run() -> Result<(), Box> { let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap(); typesetter.add_font_provider(provider); - let layouts = block_on(typesetter.typeset(&src))?; + let layouts = block_on(typesetter.typeset(&src)); let exporter = PdfExporter::new(); let writer = BufWriter::new(File::create(&dest)?); diff --git a/src/error.rs b/src/error.rs index ed43818fc..55156038f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,10 +1,20 @@ -#[derive(Debug, Clone, Eq, PartialEq)] +use serde::Serialize; + + +#[derive(Debug, Clone, Eq, PartialEq, Serialize)] pub struct Error { pub message: String, + pub severity: Severity, } impl Error { - pub fn new(message: impl Into) -> Error { - Error { message: message.into() } + pub fn new(message: impl Into, severity: Severity) -> Error { + Error { message: message.into(), severity } } } + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize)] +pub enum Severity { + Warning, + Error, +} diff --git a/src/func/macros.rs b/src/func/macros.rs index 1e92d7353..92b6c16e7 100644 --- a/src/func/macros.rs +++ b/src/func/macros.rs @@ -22,7 +22,7 @@ macro_rules! function { // Parse trait. (@parse($($a:tt)*) parse(default) $($r:tt)*) => { - function!(@parse($($a)*) parse() { Default::default() } $($r)*); + function!(@parse($($a)*) parse(_h, _b, _c, _e, _d, _m) {Default::default() } $($r)*); }; (@parse($($a:tt)*) parse($h:ident, $b:ident, $c:ident, $e:ident, $d:ident) $($r:tt)* ) => { function!(@parse($($a)*) parse($h, $b, $c, $e, $d, _metadata) $($r)*); @@ -40,7 +40,7 @@ macro_rules! function { fn parse( #[allow(unused)] mut $header: FuncHeader, - #[allow(unused)] $body: Option>, + #[allow(unused)] $body: Option<(Position, &str)>, #[allow(unused)] $ctx: ParseContext, #[allow(unused)] $metadata: Self::Meta, ) -> Parsed where Self: Sized { @@ -85,7 +85,7 @@ macro_rules! function { macro_rules! body { (opt: $body:expr, $ctx:expr, $errors:expr, $decos:expr) => ({ $body.map(|body| { - let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx); + let parsed = $crate::syntax::parse(body.0, body.1, $ctx); $errors.extend(parsed.errors); $decos.extend(parsed.decorations); parsed.output @@ -99,13 +99,19 @@ macro_rules! body { }; } -/// Construct a spanned error. +/// Construct an error with an optional span. #[macro_export] macro_rules! err { - ($span:expr, $($args:tt)*) => { - $crate::syntax::Spanned { - v: $crate::error::Error::new(format!($($args)*)), - span: $span, + (@$severity:ident: $span:expr; $($args:tt)*) => { + $crate::syntax::Spanned { v: err!(@Error: $($args)*), span: $span } + }; + + (@$severity:ident: $($args:tt)*) => { + $crate::error::Error { + message: format!($($args)*), + severity: $crate::error::Severity::$severity, } }; + + ($($tts:tt)*) => { err!(@Error: $($tts)*) }; } diff --git a/src/func/mod.rs b/src/func/mod.rs index 1ca226c38..e33999039 100644 --- a/src/func/mod.rs +++ b/src/func/mod.rs @@ -1,6 +1,5 @@ //! Dynamic typesetting functions. -use std::any::Any; use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; @@ -12,6 +11,7 @@ mod macros; /// Useful imports for creating your own functions. pub mod prelude { pub use super::{Scope, Parse, Command, Commands}; + pub use crate::error::Error; pub use crate::layout::prelude::*; pub use crate::syntax::prelude::*; pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize}; @@ -26,7 +26,7 @@ pub trait Parse { /// Parse the header and body into this function given a context. fn parse( header: FuncHeader, - body: Option>, + body: Option<(Position, &str)>, ctx: ParseContext, metadata: Self::Meta, ) -> Parsed where Self: Sized; @@ -36,7 +36,7 @@ pub trait Parse { /// implements [`Model`]. type Parser = dyn Fn( FuncHeader, - Option>, + Option<(Position, &str)>, ParseContext, ) -> Parsed>; @@ -102,11 +102,16 @@ impl Scope { } /// Return the parser with the given name if there is one. - pub(crate) fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { + pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { self.parsers.get(name) .map(|x| &**x) .ok_or_else(|| &*self.fallback) } + + /// Return the fallback parser. + pub fn get_fallback_parser(&self) -> &Parser { + &*self.fallback + } } impl Debug for Scope { diff --git a/src/layout/mod.rs b/src/layout/mod.rs index 75d34409b..1cc16a268 100644 --- a/src/layout/mod.rs +++ b/src/layout/mod.rs @@ -5,7 +5,7 @@ use smallvec::SmallVec; use toddle::query::{SharedFontLoader, FontIndex}; use crate::error::Error; -use crate::syntax::SpanVec; +use crate::syntax::{SyntaxModel, SpanVec}; use crate::size::{Size, Size2D, SizeBox}; use crate::style::LayoutStyle; @@ -26,7 +26,7 @@ pub mod prelude { /// Different kinds of layouters (fully re-exported). pub mod layouters { - pub use super::model::layout; + pub use super::model::ModelLayouter; pub use super::line::{LineLayouter, LineContext}; pub use super::stack::{StackLayouter, StackContext}; pub use super::text::{layout_text, TextContext}; @@ -37,20 +37,6 @@ pub use self::layouters::*; pub use self::prelude::*; -pub struct Layouted { - pub output: T, - pub errors: SpanVec, -} - -impl Layouted { - pub fn map(self, f: F) -> Layouted where F: FnOnce(T) -> U { - Layouted { - output: f(self.output), - errors: self.errors, - } - } -} - /// A collection of layouts. pub type MultiLayout = Vec; @@ -80,34 +66,6 @@ impl Layout { } } -/// Layout components that can be serialized. -pub trait Serialize { - /// Serialize the data structure into an output writable. - fn serialize(&self, f: &mut W) -> io::Result<()>; -} - -impl Serialize for Layout { - fn serialize(&self, f: &mut W) -> io::Result<()> { - writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?; - writeln!(f, "{}", self.actions.len())?; - for action in &self.actions { - action.serialize(f)?; - writeln!(f)?; - } - Ok(()) - } -} - -impl Serialize for MultiLayout { - fn serialize(&self, f: &mut W) -> io::Result<()> { - writeln!(f, "{}", self.len())?; - for layout in self { - layout.serialize(f)?; - } - Ok(()) - } -} - /// The general context for layouting. #[derive(Debug, Clone)] pub struct LayoutContext<'a, 'p> { @@ -133,6 +91,26 @@ pub struct LayoutContext<'a, 'p> { pub debug: bool, } +pub struct Layouted { + pub output: T, + pub errors: SpanVec, +} + +impl Layouted { + pub fn map(self, f: F) -> Layouted where F: FnOnce(T) -> U { + Layouted { + output: f(self.output), + errors: self.errors, + } + } +} + +pub async fn layout(model: &SyntaxModel, ctx: LayoutContext<'_, '_>) -> Layouted { + let mut layouter = ModelLayouter::new(ctx); + layouter.layout_syntax_model(model).await; + layouter.finish() +} + /// A possibly stack-allocated vector of layout spaces. pub type LayoutSpaces = SmallVec<[LayoutSpace; 2]>; @@ -397,3 +375,31 @@ impl LastSpacing { } } } + +/// Layout components that can be serialized. +pub trait Serialize { + /// Serialize the data structure into an output writable. + fn serialize(&self, f: &mut W) -> io::Result<()>; +} + +impl Serialize for Layout { + fn serialize(&self, f: &mut W) -> io::Result<()> { + writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?; + writeln!(f, "{}", self.actions.len())?; + for action in &self.actions { + action.serialize(f)?; + writeln!(f)?; + } + Ok(()) + } +} + +impl Serialize for MultiLayout { + fn serialize(&self, f: &mut W) -> io::Result<()> { + writeln!(f, "{}", self.len())?; + for layout in self { + layout.serialize(f)?; + } + Ok(()) + } +} diff --git a/src/layout/model.rs b/src/layout/model.rs index bcec5ceb2..73492dd90 100644 --- a/src/layout/model.rs +++ b/src/layout/model.rs @@ -1,5 +1,3 @@ -use std::pin::Pin; -use std::future::Future; use smallvec::smallvec; use crate::error::Error; @@ -9,17 +7,8 @@ use crate::syntax::{SpanVec, Spanned, Span, offset_spans}; use super::*; -pub async fn layout( - model: &SyntaxModel, - ctx: LayoutContext<'_, '_> -) -> Layouted { - let mut layouter = ModelLayouter::new(ctx); - layouter.layout_syntax_model(model).await; - layouter.finish() -} - #[derive(Debug, Clone)] -struct ModelLayouter<'a, 'p> { +pub struct ModelLayouter<'a, 'p> { ctx: LayoutContext<'a, 'p>, layouter: LineLayouter, style: LayoutStyle, @@ -28,7 +17,7 @@ struct ModelLayouter<'a, 'p> { impl<'a, 'p> ModelLayouter<'a, 'p> { /// Create a new syntax tree layouter. - fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> { + pub fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> { ModelLayouter { layouter: LineLayouter::new(LineContext { spaces: ctx.spaces.clone(), @@ -44,7 +33,7 @@ impl<'a, 'p> ModelLayouter<'a, 'p> { } } - fn layout<'r>( + pub fn layout<'r>( &'r mut self, model: Spanned<&'r dyn Model> ) -> DynFuture<'r, ()> { Box::pin(async move { @@ -60,75 +49,11 @@ impl<'a, 'p> ModelLayouter<'a, 'p> { self.errors.extend(offset_spans(layouted.errors, model.span.start)); for command in commands { - self.execute_command(command, model.span); + self.execute_command(command, model.span).await; } }) } - fn execute_command<'r>( - &'r mut self, - command: Command<'r>, - model_span: Span, - ) -> DynFuture<'r, ()> { Box::pin(async move { - use Command::*; - - match command { - LayoutSyntaxModel(model) => self.layout_syntax_model(model).await, - - Add(layout) => self.layouter.add(layout), - AddMultiple(layouts) => self.layouter.add_multiple(layouts), - SpacingFunc(space, kind, axis) => match axis { - Primary => self.layouter.add_primary_spacing(space, kind), - Secondary => self.layouter.add_secondary_spacing(space, kind), - } - - FinishLine => self.layouter.finish_line(), - FinishSpace => self.layouter.finish_space(true), - BreakParagraph => self.layout_paragraph(), - BreakPage => { - if self.ctx.nested { - self.errors.push(Spanned::new( - Error::new( "page break cannot be issued from nested context"), - model_span, - )); - } else { - self.layouter.finish_space(true) - } - } - - SetTextStyle(style) => { - self.layouter.set_line_spacing(style.line_spacing()); - self.style.text = style; - } - SetPageStyle(style) => { - if self.ctx.nested { - self.errors.push(Spanned::new( - Error::new("page style cannot be changed from nested context"), - model_span, - )); - } else { - self.style.page = style; - - let margins = style.margins(); - self.ctx.base = style.dimensions.unpadded(margins); - self.layouter.set_spaces(smallvec![ - LayoutSpace { - dimensions: style.dimensions, - padding: margins, - expansion: LayoutExpansion::new(true, true), - } - ], true); - } - } - - SetAlignment(alignment) => self.ctx.alignment = alignment, - SetAxes(axes) => { - self.layouter.set_axes(axes); - self.ctx.axes = axes; - } - } - }) } - - fn layout_syntax_model<'r>( + pub fn layout_syntax_model<'r>( &'r mut self, model: &'r SyntaxModel ) -> DynFuture<'r, ()> { Box::pin(async move { @@ -161,6 +86,73 @@ impl<'a, 'p> ModelLayouter<'a, 'p> { } }) } + pub fn finish(self) -> Layouted { + Layouted { + output: self.layouter.finish(), + errors: self.errors, + } + } + + fn execute_command<'r>( + &'r mut self, + command: Command<'r>, + span: Span, + ) -> DynFuture<'r, ()> { Box::pin(async move { + use Command::*; + + match command { + LayoutSyntaxModel(model) => self.layout_syntax_model(model).await, + + Add(layout) => self.layouter.add(layout), + AddMultiple(layouts) => self.layouter.add_multiple(layouts), + SpacingFunc(space, kind, axis) => match axis { + Primary => self.layouter.add_primary_spacing(space, kind), + Secondary => self.layouter.add_secondary_spacing(space, kind), + } + + FinishLine => self.layouter.finish_line(), + FinishSpace => self.layouter.finish_space(true), + BreakParagraph => self.layout_paragraph(), + BreakPage => { + if self.ctx.nested { + self.errors.push(err!(span; + "page break cannot be issued from nested context")); + } else { + self.layouter.finish_space(true) + } + } + + SetTextStyle(style) => { + self.layouter.set_line_spacing(style.line_spacing()); + self.style.text = style; + } + SetPageStyle(style) => { + if self.ctx.nested { + self.errors.push(err!(span; + "page style cannot be changed from nested context")); + } else { + self.style.page = style; + + let margins = style.margins(); + self.ctx.base = style.dimensions.unpadded(margins); + self.layouter.set_spaces(smallvec![ + LayoutSpace { + dimensions: style.dimensions, + padding: margins, + expansion: LayoutExpansion::new(true, true), + } + ], true); + } + } + + SetAlignment(alignment) => self.ctx.alignment = alignment, + SetAxes(axes) => { + self.layouter.set_axes(axes); + self.ctx.axes = axes; + } + } + }) } + async fn layout_text(&mut self, text: &str) { self.layouter.add(layout_text(text, TextContext { loader: &self.ctx.loader, @@ -183,11 +175,4 @@ impl<'a, 'p> ModelLayouter<'a, 'p> { SpacingKind::PARAGRAPH, ); } - - fn finish(self) -> Layouted { - Layouted { - output: self.layouter.finish(), - errors: self.errors, - } - } } diff --git a/src/lib.rs b/src/lib.rs index 2ca4d55a5..3ebe34c2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,10 +24,9 @@ use std::cell::RefCell; use smallvec::smallvec; use toddle::query::{FontLoader, FontProvider, SharedFontLoader}; -use toddle::Error as FontError; use crate::func::Scope; -use crate::layout::{Layouted, LayoutContext, MultiLayout}; +use crate::layout::{Layouted, MultiLayout}; use crate::syntax::{parse, ParseContext, Parsed, SyntaxModel, Position}; use crate::style::{LayoutStyle, PageStyle, TextStyle}; diff --git a/src/library/maps/mod.rs b/src/library/maps/mod.rs index a868ce6c7..3538def72 100644 --- a/src/library/maps/mod.rs +++ b/src/library/maps/mod.rs @@ -21,8 +21,8 @@ macro_rules! key { impl ExpressionKind for $type { const NAME: &'static str = $name; - fn from_expr(expr: Spanned) -> ParseResult { - if let Expression::Ident(ident) = expr.v { + fn from_expr(expr: Spanned) -> ParseResult { + if let Expr::Ident(ident) = expr.v { Self::from_ident(&Spanned::new(ident, expr.span)) } else { error!("expected {}", Self::NAME); @@ -55,8 +55,8 @@ impl Into> for DefaultKey { impl ExpressionKind for DefaultKey where T: ExpressionKind { const NAME: &'static str = T::NAME; - fn from_expr(expr: Spanned) -> ParseResult> { - if let Expression::Ident(ident) = &expr.v { + fn from_expr(expr: Spanned) -> ParseResult> { + if let Expr::Ident(ident) = &expr.v { match ident.as_str() { "default" => return Ok(DefaultKey::None), _ => {}, diff --git a/src/library/mod.rs b/src/library/mod.rs index d91f1b35b..8d16ccaf5 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -63,8 +63,8 @@ function! { body: parse!(optional: body, ctx), list: { header.args.iter_pos().map(|arg| match arg.v { - Expression::Str(s) | - Expression::Ident(Ident(s)) => Ok(s.to_lowercase()), + Expr::Str(s) | + Expr::Ident(Ident(s)) => Ok(s.to_lowercase()), _ => error!("expected identifier or string"), }).collect::>>()? } @@ -117,8 +117,8 @@ function! { parse(header, body, ctx) { FontWeightFunc { body: parse!(optional: body, ctx), - weight: match header.args.get_pos::()? { - Expression::Number(weight) => { + weight: match header.args.get_pos::()? { + Expr::Number(weight) => { let weight = weight.round() as i16; FontWeight( if weight < 100 { 100 } @@ -126,7 +126,7 @@ function! { else { 900 } ) } - Expression::Ident(Ident(s)) => { + Expr::Ident(Ident(s)) => { match FontWeight::from_str(&s) { Some(weight) => weight, None => error!("invalid font weight: `{}`", s), @@ -263,7 +263,7 @@ function! { SpacingFunc { axis: AxisKey::Specific(axis), spacing: FSize::from_expr( - header.args.get_pos::>()? + header.args.get_pos::>()? )?, } } else { diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index 74deda465..34a1c6bf4 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -4,7 +4,7 @@ use super::*; /// An argument or return value. #[derive(Clone, PartialEq)] -pub enum Expression { +pub enum Expr { Ident(Ident), Str(String), Number(f64), @@ -14,9 +14,24 @@ pub enum Expression { Object(Object), } -impl Display for Expression { +impl Expr { + pub fn name(&self) -> &'static str { + use Expr::*; + match self { + Ident(_) => "identifier", + Str(_) => "string", + Number(_) => "number", + Size(_) => "size", + Bool(_) => "boolean", + Tuple(_) => "tuple", + Object(_) => "object", + } + } +} + +impl Display for Expr { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - use Expression::*; + use Expr::*; match self { Ident(i) => write!(f, "{}", i), Str(s) => write!(f, "{:?}", s), @@ -29,6 +44,8 @@ impl Display for Expression { } } +debug_display!(Expr); + /// An identifier. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] pub struct Ident(pub String); @@ -53,10 +70,15 @@ impl Display for Ident { } } +debug_display!(Ident); + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct StringLike(pub String); + /// A sequence of expressions. #[derive(Clone, PartialEq)] pub struct Tuple { - pub items: Vec>, + pub items: Vec>, } impl Tuple { @@ -64,7 +86,7 @@ impl Tuple { Tuple { items: vec![] } } - pub fn add(&mut self, item: Spanned) { + pub fn add(&mut self, item: Spanned) { self.items.push(item); } } @@ -86,6 +108,8 @@ impl Display for Tuple { } } +debug_display!(Tuple); + /// A key-value collection of identifiers and associated expressions. #[derive(Clone, PartialEq)] pub struct Object { @@ -97,7 +121,7 @@ impl Object { Object { pairs: vec![] } } - pub fn add(&mut self, key: Spanned, value: Spanned) { + pub fn add(&mut self, key: Spanned, value: Spanned) { self.pairs.push(Pair { key, value }); } @@ -127,11 +151,13 @@ impl Display for Object { } } +debug_display!(Object); + /// A key-value pair in an object. #[derive(Clone, PartialEq)] pub struct Pair { pub key: Spanned, - pub value: Spanned, + pub value: Spanned, } impl Display for Pair { @@ -140,57 +166,56 @@ impl Display for Pair { } } -debug_display!(Ident); -debug_display!(Expression); -debug_display!(Tuple); -debug_display!(Object); debug_display!(Pair); -/// Kinds of expressions. -pub trait ExpressionKind: Sized { +pub trait ExprKind: Sized { /// The name of the expression in an `expected ` error. const NAME: &'static str; /// Create from expression. - fn from_expr(expr: Spanned) -> ParseResult; + fn from_expr(expr: Spanned) -> Result; } +impl ExprKind for Spanned where T: ExprKind { + const NAME: &'static str = T::NAME; + + fn from_expr(expr: Spanned) -> Result { + let span = expr.span; + T::from_expr(expr).map(|v| Spanned { v, span }) + } +} /// Implements the expression kind trait for a type. macro_rules! kind { - ($type:ty, $name:expr, $($patterns:tt)*) => { - impl ExpressionKind for $type { + ($type:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => { + impl ExprKind for $type { const NAME: &'static str = $name; - fn from_expr(expr: Spanned) -> ParseResult { + fn from_expr(expr: Spanned) -> Result { #[allow(unreachable_patterns)] Ok(match expr.v { - $($patterns)*, - _ => error!("expected {}", Self::NAME), + $($p => $r),*, + _ => return Err( + err!("expected {}, found {}", Self::NAME, expr.v.name()) + ), }) } } }; } -kind!(Expression, "expression", e => e); -kind!(Ident, "identifier", Expression::Ident(ident) => ident); -kind!(String, "string", Expression::Str(string) => string); -kind!(f64, "number", Expression::Number(num) => num); -kind!(bool, "boolean", Expression::Bool(boolean) => boolean); -kind!(Size, "size", Expression::Size(size) => size); -kind!(Tuple, "tuple", Expression::Tuple(tuple) => tuple); -kind!(Object, "object", Expression::Object(object) => object); - -kind!(ScaleSize, "number or size", - Expression::Size(size) => ScaleSize::Absolute(size), - Expression::Number(scale) => ScaleSize::Scaled(scale as f32) +kind!(Expr, "expression", e => e); +kind!(Ident, "identifier", Expr::Ident(i) => i); +kind!(String, "string", Expr::Str(s) => s); +kind!(f64, "number", Expr::Number(n) => n); +kind!(bool, "boolean", Expr::Bool(b) => b); +kind!(Size, "size", Expr::Size(s) => s); +kind!(Tuple, "tuple", Expr::Tuple(t) => t); +kind!(Object, "object", Expr::Object(o) => o); +kind!(ScaleSize, "number or size", + Expr::Size(size) => ScaleSize::Absolute(size), + Expr::Number(scale) => ScaleSize::Scaled(scale as f32), +); +kind!(StringLike, "identifier or string", + Expr::Ident(Ident(s)) => StringLike(s), + Expr::Str(s) => StringLike(s), ); - -impl ExpressionKind for Spanned where T: ExpressionKind { - const NAME: &'static str = T::NAME; - - fn from_expr(expr: Spanned) -> ParseResult> { - let span = expr.span; - T::from_expr(expr).map(|v| Spanned { v, span }) - } -} diff --git a/src/syntax/func.rs b/src/syntax/func.rs index 5b1ce6e88..abc8c4310 100644 --- a/src/syntax/func.rs +++ b/src/syntax/func.rs @@ -15,7 +15,7 @@ pub struct FuncArgs { #[derive(Debug, Clone, PartialEq)] pub enum Arg { - Pos(Spanned), + Pos(Spanned), Key(Pair), } @@ -46,12 +46,12 @@ impl FuncArgs { } /// Add a positional argument. - pub fn add_pos(&mut self, item: Spanned) { + pub fn add_pos(&mut self, item: Spanned) { self.pos.add(item); } /// Add a keyword argument. - pub fn add_key(&mut self, key: Spanned, value: Spanned) { + pub fn add_key(&mut self, key: Spanned, value: Spanned) { self.key.add(key, value); } @@ -92,7 +92,7 @@ impl FuncArgs { // } // /// Iterator over positional arguments. - // pub fn iter_pos(&mut self) -> std::vec::IntoIter> { + // pub fn iter_pos(&mut self) -> std::vec::IntoIter> { // let tuple = std::mem::replace(&mut self.positional, Tuple::new()); // tuple.items.into_iter() // } diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 75407f829..a77c764e9 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -23,22 +23,6 @@ pub mod prelude { } -pub struct Parsed { - pub output: T, - pub errors: SpanVec, - pub decorations: SpanVec, -} - -impl Parsed { - pub fn map(self, f: F) -> Parsed where F: FnOnce(T) -> U { - Parsed { - output: f(self.output), - errors: self.errors, - decorations: self.decorations, - } - } -} - #[async_trait::async_trait(?Send)] pub trait Model: Debug + ModelBounds { async fn layout<'a>( @@ -110,7 +94,7 @@ impl SyntaxModel { #[async_trait::async_trait(?Send)] impl Model for SyntaxModel { - async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted> { + async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted> { Layouted { output: vec![Command::LayoutSyntaxModel(self)], errors: vec![], @@ -153,7 +137,8 @@ impl PartialEq for Node { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] pub enum Decoration { ValidFuncName, InvalidFuncName, diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 24bef7ce5..ed3430502 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -3,10 +3,6 @@ use super::*; use Token::*; -pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed { - Parser::new(start, src, ctx).parse() -} - /// The context for parsing. #[derive(Debug, Copy, Clone)] pub struct ParseContext<'a> { @@ -14,185 +10,169 @@ pub struct ParseContext<'a> { pub scope: &'a Scope, } -struct Parser<'s> { - src: &'s str, - ctx: ParseContext<'s>, - tokens: Tokens<'s>, - peeked: Option>>>, - position: Position, - last_position: Position, - errors: SpanVec, - decorations: SpanVec, +pub struct Parsed { + pub output: T, + pub errors: SpanVec, + pub decorations: SpanVec, } -impl<'s> Parser<'s> { - fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> { - Parser { - src, +impl Parsed { + pub fn map(self, f: F) -> Parsed where F: FnOnce(T) -> U { + Parsed { + output: f(self.output), + errors: self.errors, + decorations: self.decorations, + } + } +} + +pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed { + let mut model = SyntaxModel::new(); + let mut errors = Vec::new(); + let mut decorations = Vec::new(); + + let mut tokens = Tokens::new(start, src, TokenizationMode::Body); + + while let Some(token) = tokens.next() { + let span = token.span; + + let node = match token.v { + Space(newlines) => if newlines >= 2 { + Node::Newline + } else { + Node::Space + }, + + Function { header, body, terminated } => { + let parsed: Parsed = FuncParser::new(header, body, ctx).parse(); + + errors.extend(offset_spans(parsed.errors, span.start)); + decorations.extend(offset_spans(parsed.decorations, span.start)); + + if !terminated { + errors.push(err!(Span::at(span.end); "expected closing bracket")); + } + + parsed.output + } + + Star => Node::ToggleBolder, + Underscore => Node::ToggleItalic, + Backtick => Node::ToggleMonospace, + Text(text) => Node::Text(text.to_owned()), + + LineComment(_) | BlockComment(_) => continue, + + other => { + errors.push(err!(span; "unexpected {}", name(other))); + continue; + } + }; + + model.add(Spanned { v: node, span: token.span }); + } + + Parsed { output: model, errors, decorations } +} + +struct FuncParser<'s> { + ctx: ParseContext<'s>, + errors: SpanVec, + decorations: SpanVec, + tokens: Tokens<'s>, + peeked: Option>>>, + body: Option<(Position, &'s str)>, +} + +impl<'s> FuncParser<'s> { + fn new( + header: &'s str, + body: Option<(Position, &'s str)>, + ctx: ParseContext<'s> + ) -> FuncParser<'s> { + FuncParser { ctx, - tokens: tokenize(start, src), - peeked: None, - position: Position::ZERO, - last_position: Position::ZERO, errors: vec![], decorations: vec![], + tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header), + peeked: None, + body, } } - /// The main parsing entrypoint. - fn parse(mut self) -> Parsed { - let mut model = SyntaxModel::new(); - - while let Some(token) = self.eat() { - let mut span = token.span; - let node = match token.v { - LineComment(_) | BlockComment(_) => None, - Whitespace(newlines) => Some(if newlines >= 2 { - Node::Newline - } else { - Node::Space - }), - - LeftBracket => self.parse_func().map(|spanned| { - span = spanned.span; - spanned.v - }), - - Star => Some(Node::ToggleBolder), - Underscore => Some(Node::ToggleItalic), - Backtick => Some(Node::ToggleMonospace), - Text(text) => Some(Node::Text(text.to_owned())), - - _ => { - self.unexpected(token); - None + fn parse(mut self) -> Parsed { + let parsed = if let Some(header) = self.parse_func_header() { + let name = header.name.v.as_str(); + let (parser, deco) = match self.ctx.scope.get_parser(name) { + Ok(parser) => (parser, Decoration::ValidFuncName), + Err(parser) => { + self.errors.push(err!(header.name.span; "unknown function")); + (parser, Decoration::InvalidFuncName) } }; - if let Some(v) = node { - model.add(Spanned { v, span }); - } - } + self.decorations.push(Spanned::new(deco, header.name.span)); + + parser(header, self.body, self.ctx) + } else { + let default = FuncHeader { + name: Spanned::new(Ident("".to_string()), Span::ZERO), + args: FuncArgs::new(), + }; + + // Use the fallback function such that the body is still rendered + // even if the header is completely unparsable. + self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx) + }; + + self.errors.extend(parsed.errors); + self.decorations.extend(parsed.decorations); Parsed { - output: model, + output: Node::Model(parsed.output), errors: self.errors, decorations: self.decorations, } } - /// Parses a function including header and body with the cursor starting - /// right behind the first opening bracket. - fn parse_func(&mut self) -> Option> { - let start = self.last_pos(); - - let header = self.parse_func_header(); - self.eat_until(|t| t == RightBracket, false); - - if self.eat().map(Spanned::value) != Some(RightBracket) { - self.expected_at("closing bracket", self.pos()); - } - - let body = if self.peekv() == Some(LeftBracket) { - self.eat(); - - let start_index = self.tokens.index(); - let start_position = self.tokens.pos(); - - let found = self.tokens.move_to_closing_bracket(); - - let end_index = self.tokens.index(); - let end_position = self.tokens.pos(); - - let body = &self.src[start_index .. end_index]; - - self.position = end_position; - - if found { - let next = self.eat().map(Spanned::value); - debug_assert_eq!(next, Some(RightBracket)); - } else { - self.expected_at("closing bracket", self.pos()); - } - - Some(Spanned::new(body, Span::new(start_position, end_position))) - } else { - None - }; - - let header = header?; - let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) { - Ok(parser) => (parser, Decoration::ValidFuncName), - Err(parser) => { - let error = Error::new(format!("unknown function: `{}`", header.name.v)); - self.errors.push(Spanned::new(error, header.name.span)); - (parser, Decoration::InvalidFuncName) - } - }; - - self.decorations.push(Spanned::new(decoration, header.name.span)); - - let parsed = parser(header, body, self.ctx); - self.errors.extend(offset_spans(parsed.errors, start)); - self.decorations.extend(offset_spans(parsed.decorations, start)); - - let node = Node::Model(parsed.output); - - let end = self.pos(); - let span = Span { start, end }; - - Some(Spanned { v: node, span }) - } - - /// Parses a function header including the closing bracket. fn parse_func_header(&mut self) -> Option { + let start = self.pos(); self.skip_whitespace(); - let name = self.parse_func_name()?; + + let name = match self.eat() { + Some(Spanned { v: ExprIdent(ident), span }) => { + Spanned { v: Ident(ident.to_string()), span } + } + other => { + self.expected_found_or_at("identifier", other, start); + return None; + } + }; self.skip_whitespace(); - let args = match self.peek() { - Some(Spanned { v: Colon, .. }) => { - self.eat(); - self.parse_func_args() - } - Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(), - other => { - self.expected_at("colon or closing bracket", name.span.end); + let args = match self.eat().map(Spanned::value) { + Some(Colon) => self.parse_func_args(), + Some(_) => { + self.expected_at("colon", name.span.end); FuncArgs::new() } + None => FuncArgs::new(), }; Some(FuncHeader { name, args }) } - /// Parses the function name if is the next token. Otherwise, it adds an - /// error and returns `None`. - fn parse_func_name(&mut self) -> Option> { - match self.peek() { - Some(Spanned { v: ExprIdent(ident), span }) => { - self.eat(); - return Some(Spanned { v: Ident(ident.to_string()), span }); - } - other => self.expected_found_or_at("identifier", other, self.pos()), - } - - None - } - - /// Parses the function arguments and stops right before the final closing - /// bracket. fn parse_func_args(&mut self) -> FuncArgs { let mut args = FuncArgs::new(); - loop { - self.skip_whitespace(); - match self.peekv() { - Some(RightBracket) | None => break, - _ => match self.parse_arg() { - Some(arg) => args.add(arg), - None => {} - } + self.skip_whitespace(); + while self.peek().is_some() { + match self.parse_arg() { + Some(arg) => args.add(arg), + None => {} } + + self.skip_whitespace(); } args @@ -221,7 +201,7 @@ impl<'s> Parser<'s> { }) }) } else { - Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span))) + Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span))) } } else { self.parse_expr().map(|expr| Arg::Pos(expr)) @@ -230,7 +210,6 @@ impl<'s> Parser<'s> { if let Some(arg) = &arg { self.skip_whitespace(); match self.peekv() { - Some(RightBracket) => {} Some(Comma) => { self.eat(); } Some(_) => self.expected_at("comma", arg.span().end), _ => {} @@ -244,19 +223,27 @@ impl<'s> Parser<'s> { } /// Parse a atomic or compound (tuple / object) expression. - fn parse_expr(&mut self) -> Option> { + fn parse_expr(&mut self) -> Option> { let first = self.peek()?; - let mut expr = |v| { - self.eat(); - Spanned { v, span: first.span } - }; + let spanned = |v| Spanned { v, span: first.span }; Some(match first.v { - ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))), - ExprStr(s) => expr(Expression::Str(s.to_string())), - ExprNumber(n) => expr(Expression::Number(n)), - ExprSize(s) => expr(Expression::Size(s)), - ExprBool(b) => expr(Expression::Bool(b)), + ExprIdent(i) => { + self.eat(); + spanned(Expr::Ident(Ident(i.to_string()))) + } + ExprStr { string, terminated } => { + if !terminated { + self.expected_at("quote", first.span.end); + } + + self.eat(); + spanned(Expr::Str(string.to_string())) + } + ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) } + ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) } + ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) } + LeftParen => self.parse_tuple(), LeftBrace => self.parse_object(), _ => return None, @@ -264,56 +251,48 @@ impl<'s> Parser<'s> { } /// Parse a tuple expression. - fn parse_tuple(&mut self) -> Spanned { + fn parse_tuple(&mut self) -> Spanned { let start = self.pos(); // TODO: Do the thing. - self.eat_until(|t| matches!(t, RightParen | RightBracket), false); - if self.peekv() == Some(RightParen) { - self.eat(); - } + self.eat_until(|t| t == RightParen, true); let end = self.pos(); let span = Span { start, end }; - Spanned { v: Expression::Tuple(Tuple::new()), span } + Spanned { v: Expr::Tuple(Tuple::new()), span } } /// Parse an object expression. - fn parse_object(&mut self) -> Spanned { + fn parse_object(&mut self) -> Spanned { let start = self.pos(); // TODO: Do the thing. - self.eat_until(|t| matches!(t, RightBrace | RightBracket), false); - if self.peekv() == Some(RightBrace) { - self.eat(); - } + self.eat_until(|t| t == RightBrace, true); let end = self.pos(); let span = Span { start, end }; - Spanned { v: Expression::Object(Object::new()), span } + Spanned { v: Expr::Object(Object::new()), span } } /// Skip all whitespace/comment tokens. fn skip_whitespace(&mut self) { self.eat_until(|t| - !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false) - } - - /// Add an error about an `thing` which was expected but not found at the - /// given position. - fn expected_at(&mut self, thing: &str, pos: Position) { - let error = Error::new(format!("expected {}", thing)); - self.errors.push(Spanned::new(error, Span::at(pos))); + !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false) } /// Add an error about an expected `thing` which was not found, showing /// what was found instead. fn expected_found(&mut self, thing: &str, found: Spanned) { - let message = format!("expected {}, found {}", thing, name(found.v)); - let error = Error::new(message); - self.errors.push(Spanned::new(error, found.span)); + self.errors.push(err!(found.span; + "expected {}, found {}", thing, name(found.v))); + } + + /// Add an error about an `thing` which was expected but not found at the + /// given position. + fn expected_at(&mut self, thing: &str, pos: Position) { + self.errors.push(err!(Span::at(pos); "expected {}", thing)); } /// Add a found-error if `found` is some and a positional error, otherwise. @@ -329,12 +308,6 @@ impl<'s> Parser<'s> { } } - /// Add an error about an unexpected token `found`. - fn unexpected(&mut self, found: Spanned) { - let error = Error::new(format!("unexpected {}", name(found.v))); - self.errors.push(Spanned::new(error, found.span)); - } - /// Consume tokens until the function returns true and only consume the last /// token if instructed to. fn eat_until(&mut self, mut f: F, eat_match: bool) @@ -351,19 +324,10 @@ impl<'s> Parser<'s> { } } - /// Consume and return the next token, update positions and colorize the - /// token. All colorable tokens are per default colorized here, to override - /// a colorization use `Colorization::replace_last`. + /// Consume and return the next token. fn eat(&mut self) -> Option>> { - let token = self.peeked.take() - .unwrap_or_else(|| self.tokens.next()); - - if let Some(token) = token { - self.last_position = self.position; - self.position = token.span.end; - } - - token + self.peeked.take() + .unwrap_or_else(|| self.tokens.next()) } /// Peek at the next token without consuming it. @@ -379,23 +343,18 @@ impl<'s> Parser<'s> { /// The position at the end of the last eat token / start of the peekable /// token. fn pos(&self) -> Position { - self.position - } - - /// The position at the start of the last eaten token. - fn last_pos(&self) -> Position { - self.last_position + self.peeked.flatten() + .map(|s| s.span.start) + .unwrap_or_else(|| self.tokens.pos()) } } -/// The name of a token in an `expected <...>` error. +/// The name of a token in an `(un)expected <...>` error. fn name(token: Token) -> &'static str { match token { - Whitespace(_) => "whitespace", + Space(_) => "space", LineComment(_) | BlockComment(_) => "comment", - StarSlash => "end of block comment", - LeftBracket => "opening bracket", - RightBracket => "closing bracket", + Function { .. } => "function", LeftParen => "opening paren", RightParen => "closing paren", LeftBrace => "opening brace", @@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str { Comma => "comma", Equals => "equals sign", ExprIdent(_) => "identifier", - ExprStr(_) => "string", + ExprStr { .. } => "string", ExprNumber(_) => "number", ExprSize(_) => "size", - ExprBool(_) => "bool", + ExprBool(_) => "boolean", Star => "star", Underscore => "underscore", Backtick => "backtick", Text(_) => "invalid identifier", + Invalid("]") => "closing bracket", + Invalid("*/") => "end of block comment", + Invalid(_) => "invalid token", } } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index eb39677e7..e049861fe 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -1,7 +1,7 @@ //! Spans map elements to the part of source code they originate from. use std::fmt::{self, Debug, Display, Formatter}; -use std::ops::{Add, AddAssign}; +use std::ops::{Add, Sub}; use serde::Serialize; @@ -21,12 +21,13 @@ impl Spanned { self.v } - pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { + pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { Spanned { v: f(self.v), span: self.span } } - pub fn map_v(&self, new_v: V) -> Spanned { - Spanned { v: new_v, span: self.span } + pub fn map_span(mut self, f: F) -> Spanned where F: FnOnce(Span) -> Span { + self.span = f(self.span); + self } } @@ -74,6 +75,13 @@ impl Span { pub fn expand(&mut self, other: Span) { *self = Span::merge(*self, other) } + + pub fn offset(self, start: Position) -> Span { + Span { + start: start + self.start, + end: start + self.end, + } + } } impl Display for Span { @@ -119,9 +127,21 @@ impl Add for Position { } } -impl AddAssign for Position { - fn add_assign(&mut self, other: Position) { - *self = *self + other; +impl Sub for Position { + type Output = Position; + + fn sub(self, rhs: Position) -> Position { + if self.line == rhs.line { + Position { + line: 0, + column: self.column - rhs.column + } + } else { + Position { + line: self.line - rhs.line, + column: self.column, + } + } } } @@ -136,14 +156,6 @@ debug_display!(Position); /// A vector of spanned things. pub type SpanVec = Vec>; -pub fn offset_spans( - vec: SpanVec, - start: Position, -) -> impl Iterator> { - vec.into_iter() - .map(move |mut spanned| { - spanned.span.start += start; - spanned.span.end += start; - spanned - }) +pub fn offset_spans(vec: SpanVec, start: Position) -> impl Iterator> { + vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 0588bc6c9..6c8e736c3 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID; use super::*; use Token::*; -use State::*; +use TokenizationMode::*; /// A minimal semantic entity of source code. @@ -12,20 +12,20 @@ use State::*; pub enum Token<'s> { /// One or more whitespace characters. The contained `usize` denotes the /// number of newlines that were contained in the whitespace. - Whitespace(usize), + Space(usize), /// A line comment with inner string contents `//<&'s str>\n`. LineComment(&'s str), /// A block comment with inner string contents `/*<&'s str>*/`. The comment /// can contain nested block comments. BlockComment(&'s str), - /// An erroneous `*/` without an opening block comment. - StarSlash, - /// A left bracket: `[`. - LeftBracket, - /// A right bracket: `]`. - RightBracket, + /// A function invocation `[
][]`. + Function { + header: &'s str, + body: Option<(Position, &'s str)>, + terminated: bool, + }, /// A left parenthesis in a function header: `(`. LeftParen, @@ -46,7 +46,7 @@ pub enum Token<'s> { /// An identifier in a function header: `center`. ExprIdent(&'s str), /// A quoted string in a function header: `"..."`. - ExprStr(&'s str), + ExprStr { string: &'s str, terminated: bool }, /// A number in a function header: `3.14`. ExprNumber(f64), /// A size in a function header: `12pt`. @@ -63,36 +63,31 @@ pub enum Token<'s> { /// Any other consecutive string. Text(&'s str), -} -/// Decomposes text into a sequence of semantic tokens. -pub fn tokenize(start: Position, src: &str) -> Tokens { - Tokens::new(start, src) + /// Things that are not valid in the context they appeared in. + Invalid(&'s str), } /// An iterator over the tokens of a string of source code. pub struct Tokens<'s> { src: &'s str, - state: State, - stack: Vec<(State, Position)>, + mode: TokenizationMode, iter: Peekable>, position: Position, index: usize, } #[derive(Debug, Copy, Clone, Eq, PartialEq)] -enum State { +pub enum TokenizationMode { Header, - StartBody, Body, } impl<'s> Tokens<'s> { - pub fn new(start: Position, src: &'s str) -> Tokens<'s> { + pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> { Tokens { src, - state: State::Body, - stack: vec![], + mode, iter: src.chars().peekable(), position: start, index: 0, @@ -110,35 +105,6 @@ impl<'s> Tokens<'s> { pub fn pos(&self) -> Position { self.position } - - /// Move through the string until an unbalanced closing bracket is found - /// without tokenizing the contents. - /// - /// Returns whether a closing bracket was found or the end of the string was - /// reached. - pub fn move_to_closing_bracket(&mut self) -> bool { - let mut escaped = false; - let mut depth = 0; - - self.read_string_until(|n| { - match n { - '[' if !escaped => depth += 1, - ']' if !escaped => { - if depth == 0 { - return true; - } else { - depth -= 1; - } - } - '\\' => escaped = !escaped, - _ => escaped = false, - } - - false - }, false, 0, 0); - - self.peek() == Some(']') - } } impl<'s> Iterator for Tokens<'s> { @@ -153,55 +119,31 @@ impl<'s> Iterator for Tokens<'s> { // Comments. '/' if self.peek() == Some('/') => self.parse_line_comment(), '/' if self.peek() == Some('*') => self.parse_block_comment(), - '*' if self.peek() == Some('/') => { self.eat(); StarSlash } + '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") } // Whitespace. c if c.is_whitespace() => self.parse_whitespace(start), // Functions. - '[' => { - match self.state { - Header | Body => { - self.stack.push((self.state, start)); - self.position = Position::new(0, '['.len_utf8()); - self.state = Header; - } - StartBody => self.state = Body, - } - - LeftBracket - } - ']' => { - if self.state == Header && self.peek() == Some('[') { - self.state = StartBody; - } else { - if let Some((state, pos)) = self.stack.pop() { - self.state = state; - self.position = pos + self.position; - } else { - self.state = Body; - } - } - - RightBracket - } + '[' => self.parse_function(start), + ']' => Invalid("]"), // Syntactic elements in function headers. - '(' if self.state == Header => LeftParen, - ')' if self.state == Header => RightParen, - '{' if self.state == Header => LeftBrace, - '}' if self.state == Header => RightBrace, - ':' if self.state == Header => Colon, - ',' if self.state == Header => Comma, - '=' if self.state == Header => Equals, + '(' if self.mode == Header => LeftParen, + ')' if self.mode == Header => RightParen, + '{' if self.mode == Header => LeftBrace, + '}' if self.mode == Header => RightBrace, + ':' if self.mode == Header => Colon, + ',' if self.mode == Header => Comma, + '=' if self.mode == Header => Equals, // String values. - '"' if self.state == Header => self.parse_string(), + '"' if self.mode == Header => self.parse_string(), // Style toggles. - '*' if self.state == Body => Star, - '_' if self.state == Body => Underscore, - '`' if self.state == Body => Backtick, + '*' if self.mode == Body => Star, + '_' if self.mode == Body => Underscore, + '`' if self.mode == Body => Backtick, // An escaped thing. '\\' => self.parse_escaped(), @@ -215,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> { ',' | '"' | '/' => true, _ => false, } - }, false, -(c.len_utf8() as isize), 0); + }, false, -(c.len_utf8() as isize), 0).0; - if self.state == Header { + if self.mode == Header { self.parse_expr(text) } else { Text(text) @@ -234,7 +176,7 @@ impl<'s> Iterator for Tokens<'s> { impl<'s> Tokens<'s> { fn parse_line_comment(&mut self) -> Token<'s> { - LineComment(self.read_string_until(is_newline_char, false, 1, 0)) + LineComment(self.read_string_until(is_newline_char, false, 1, 0).0) } fn parse_block_comment(&mut self) -> Token<'s> { @@ -262,19 +204,60 @@ impl<'s> Tokens<'s> { } false - }, true, 0, -2)) + }, true, 0, -2).0) } fn parse_whitespace(&mut self, start: Position) -> Token<'s> { self.read_string_until(|n| !n.is_whitespace(), false, 0, 0); let end = self.pos(); - Whitespace(end.line - start.line) + Space(end.line - start.line) + } + + fn parse_function(&mut self, start: Position) -> Token<'s> { + let (header, terminated) = self.read_function_part(); + self.eat(); + + if self.peek() != Some('[') { + return Function { header, body: None, terminated }; + } + + self.eat(); + + let offset = self.pos() - start; + let (body, terminated) = self.read_function_part(); + self.eat(); + + Function { header, body: Some((offset, body)), terminated } + } + + fn read_function_part(&mut self) -> (&'s str, bool) { + let mut escaped = false; + let mut in_string = false; + let mut depth = 0; + + self.read_string_until(|n| { + match n { + '"' if !escaped => in_string = !in_string, + '[' if !escaped && !in_string => depth += 1, + ']' if !escaped && !in_string => { + if depth == 0 { + return true; + } else { + depth -= 1; + } + } + '\\' => escaped = !escaped, + _ => escaped = false, + } + + false + }, false, 0, 0) } fn parse_string(&mut self) -> Token<'s> { let mut escaped = false; - ExprStr(self.read_string_until(|n| { + let (string, terminated) = self.read_string_until(|n| { match n { '"' if !escaped => return true, '\\' => escaped = !escaped, @@ -282,7 +265,8 @@ impl<'s> Tokens<'s> { } false - }, true, 0, -1)) + }, true, 0, -1); + ExprStr { string, terminated } } fn parse_escaped(&mut self) -> Token<'s> { @@ -294,7 +278,7 @@ impl<'s> Tokens<'s> { } let c = self.peek().unwrap_or('n'); - if self.state == Body && is_escapable(c) { + if self.mode == Body && is_escapable(c) { let index = self.index(); self.eat(); Text(&self.src[index .. index + c.len_utf8()]) @@ -315,7 +299,7 @@ impl<'s> Tokens<'s> { } else if is_identifier(text) { ExprIdent(text) } else { - Text(text) + Invalid(text) } } @@ -325,7 +309,7 @@ impl<'s> Tokens<'s> { eat_match: bool, offset_start: isize, offset_end: isize, - ) -> &'s str where F: FnMut(char) -> bool { + ) -> (&'s str, bool) where F: FnMut(char) -> bool { let start = ((self.index() as isize) + offset_start) as usize; let mut matched = false; @@ -346,7 +330,7 @@ impl<'s> Tokens<'s> { end = ((end as isize) + offset_end) as usize; } - &self.src[start .. end] + (&self.src[start .. end], matched) } fn eat(&mut self) -> Option { diff --git a/tests/layouter/test.typ b/tests/layouter/test.typ deleted file mode 100644 index b7fd334d1..000000000 --- a/tests/layouter/test.typ +++ /dev/null @@ -1 +0,0 @@ -[box][hi] diff --git a/tests/parser/tokens.rs b/tests/parser/tokens.rs index fb48b32e9..66e44ac57 100644 --- a/tests/parser/tokens.rs +++ b/tests/parser/tokens.rs @@ -1,80 +1,77 @@ // Whitespace. t "" => [] -t " " => [W(0)] -t " " => [W(0)] -t "\t" => [W(0)] -t " \t" => [W(0)] -t "\n" => [W(1)] -t "\n " => [W(1)] -t " \n" => [W(1)] -t " \n " => [W(1)] -t " \n\t \n " => [W(2)] -t "\r\n" => [W(1)] -t " \r\r\n \x0D" => [W(3)] -t "\n\r" => [W(2)] +t " " => [S(0)] +t " " => [S(0)] +t "\t" => [S(0)] +t " \t" => [S(0)] +t "\n" => [S(1)] +t "\n " => [S(1)] +t " \n" => [S(1)] +t " \n " => [S(1)] +t " \n\t \n " => [S(2)] +t "\r\n" => [S(1)] +t " \r\r\n \x0D" => [S(3)] +t "\n\r" => [S(2)] // Comments. -t "a // bc\n " => [T("a"), W(0), LC(" bc"), W(1)] -t "a //a//b\n " => [T("a"), W(0), LC("a//b"), W(1)] -t "a //a//b\r\n" => [T("a"), W(0), LC("a//b"), W(1)] -t "a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")] +t "a // bc\n " => [T("a"), S(0), LC(" bc"), S(1)] +t "a //a//b\n " => [T("a"), S(0), LC("a//b"), S(1)] +t "a //a//b\r\n" => [T("a"), S(0), LC("a//b"), S(1)] +t "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")] t "/**/" => [BC("")] -t "_/*_/*a*/*/" => [U, BC("_/*a*/")] +t "_/*_/*a*/*/" => [Underscore, BC("_/*a*/")] t "/*/*/" => [BC("/*/")] -t "abc*/" => [T("abc"), SS] +t "abc*/" => [T("abc"), Invalid("*/")] // Header only tokens. -t "[" => [LB] -t "]" => [RB] -t "[(){}:=,]" => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB] -t "[a:b]" => [LB, ID("a"), CL, ID("b"), RB] -t "[🌓, 🌍,]" => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB] -t "[=]" => [LB, EQ, RB] -t "[,]" => [LB, CM, RB] -t "a: b" => [T("a"), T(":"), W(0), T("b")] -t "c=d, " => [T("c"), T("=d"), T(","), W(0)] -t r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB] -t r#"["hi", 12pt]"# => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB] -t "\"hi\"" => [T("\"hi"), T("\"")] -t "[a: true, x=1]" => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0), - ID("x"), EQ, NUM(1.0), RB] -t "[120%]" => [LB, NUM(1.2), RB] +th "[" => [Func("", None, false)] +th "]" => [Invalid("]")] +th "(){}:=," => [LP, RP, LB, RB, Colon, Equals, Comma] +th "a:b" => [Id("a"), Colon, Id("b")] +th "=" => [Equals] +th "," => [Comma] +th r#""hello\"world""# => [Str(r#"hello\"world"#)] +th r#""hi", 12pt"# => [Str("hi"), Comma, S(0), Size(12.0)] +th "\"hi\"" => [T("\"hi"), T("\"")] +th "a: true, x=1" => [Id("a"), Colon, S(0), Bool(true), Comma, S(0), + Id("x"), Equals, Num(1.0)] +th "120%" => [Num(1.2)] +th "🌓, 🌍," => [T("🌓"), Comma, S(0), T("🌍"), Comma] +tb "a: b" => [T("a"), T(":"), S(0), T("b")] +tb "c=d, " => [T("c"), T("=d"), T(","), S(0)] // Body only tokens. -t "_*`" => [U, S, B] -t "[func]*bold*" => [LB, ID("func"), RB, S, T("bold"), S] -t "[_*`]" => [LB, T("_"), T("*"), T("`"), RB] -t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")] +tb "_*`" => [Underscore, Star, Backtick] +tb "[func]*bold*" => [Func("func", None, true), Star, T("bold"), Star] +tb "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")] +th "_*`" => [Invalid("_"), Invalid("*"), Invalid("`")] // Nested functions. -t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB] -t "[_][[,],]," => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")] -t "[=][=][=]" => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB] -t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB] +tb "[f: [=][*]]" => [Func("f: [=][*]", None, true)] +tb "[_][[,],]," => [Func("_", Some("[,],"), true), T(",")] +tb "[=][=][=]" => [Func("=", Some("="), true), Func("=", None, true)] +tb "[=][[=][=][=]]" => [Func("=", Some("[=][=][=]")), true] // Escapes. -t r"\[" => [T("[")] -t r"\]" => [T("]")] -t r"\\" => [T(r"\")] -t r"\/" => [T("/")] -t r"\*" => [T("*")] -t r"\_" => [T("_")] -t r"\`" => [T("`")] +tb r"\[" => [T("[")] +tb r"\]" => [T("]")] +tb r"\\" => [T(r"\")] +tb r"\/" => [T("/")] +tb r"\*" => [T("*")] +tb r"\_" => [T("_")] +tb r"\`" => [T("`")] // Unescapable special symbols. -t r"\:" => [T(r"\"), T(":")] -t r"\=" => [T(r"\"), T("=")] -t r"[\:]" => [LB, T(r"\"), CL, RB] -t r"[\=]" => [LB, T(r"\"), EQ, RB] -t r"[\,]" => [LB, T(r"\"), CM, RB] +th r"\:" => [T(r"\"), T(":")] +th r"\=" => [T(r"\"), T("=")] +th r"\:" => [T(r"\"), Colon] +th r"\=" => [T(r"\"), Equals] +th r"\," => [T(r"\"), Comma] -// Spans -ts "hello" => [(0:0, 0:5, T("hello"))] -ts "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, W(1)), (1:0, 1:1, T("c"))] -ts "[a=10]" => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ), - (0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)] -ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)), - (0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")), - (0:10, 0:11, RB), (0:11, 0:12, S)] -ts "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))] -ts "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)] +// Spans. +tbs "hello" => [(0:0, 0:5, T("hello"))] +tbs "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))] +tbs "[x = \"(1)\"]*" => [(0:0, 0:11, Func("x = \"(1)\"", None, true)), (0:11, 0:12, Star)] +tbs "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))] +tbs "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)] +ths "a=10" => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))] diff --git a/tests/src/layouter.rs b/tests/src/layouter.rs index eeca1a1bb..95eea204d 100644 --- a/tests/src/layouter.rs +++ b/tests/src/layouter.rs @@ -10,8 +10,8 @@ use futures_executor::block_on; use typstc::Typesetter; use typstc::layout::{MultiLayout, Serialize}; -use typstc::size::{Size, Size2D}; -use typstc::style::PageStyle; +use typstc::size::{Size, Size2D, ValueBox}; +use typstc::style::{PageStyle, PaperClass}; use typstc::toddle::query::FileSystemFontProvider; use typstc::export::pdf::PdfExporter; @@ -62,14 +62,15 @@ fn main() -> DynResult<()> { Ok(()) } -/// Create a _PDF_ with a name from the source code. +/// Create a _PDF_ and render with a name from the source code. fn test(name: &str, src: &str) -> DynResult<()> { println!("Testing: {}.", name); let mut typesetter = Typesetter::new(); typesetter.set_page_style(PageStyle { + class: PaperClass::Custom, dimensions: Size2D::with_all(Size::pt(250.0)), - .. PageStyle::default() + margins: ValueBox::with_all(None), }); let provider = FileSystemFontProvider::from_index("../fonts/index.json")?; @@ -120,31 +121,43 @@ fn test(name: &str, src: &str) -> DynResult<()> { /// Compile the source code with the typesetter. fn compile(typesetter: &Typesetter, src: &str) -> MultiLayout { - #[cfg(not(debug_assertions))] { - use std::time::Instant; + #![allow(unused_variables)] + use std::time::Instant; - // Warmup. + // Warmup. + #[cfg(not(debug_assertions))] + let warmup = { let warmup_start = Instant::now(); block_on(typesetter.typeset(&src)); - let warmup_end = Instant::now(); + Instant::now() - warmup_start + }; - let start = Instant::now(); - let tree = typesetter.parse(&src).output; - let mid = Instant::now(); - let layouts = block_on(typesetter.layout(&tree)).output; - let end = Instant::now(); + let start = Instant::now(); + let parsed = typesetter.parse(&src); + let parse = Instant::now() - start; - println!(" - cold start: {:?}", warmup_end - warmup_start); - println!(" - warmed up: {:?}", end - start); - println!(" - parsing: {:?}", mid - start); - println!(" - layouting: {:?}", end - mid); - println!(); - - layouts + if !parsed.errors.is_empty() { + println!("parse errors: {:#?}", parsed.errors); } - #[cfg(debug_assertions)] - block_on(typesetter.typeset(&src)) + let start_layout = Instant::now(); + let layouted = block_on(typesetter.layout(&parsed.output)); + let layout = Instant::now() - start_layout; + let total = Instant::now() - start; + + if !layouted.errors.is_empty() { + println!("layout errors: {:#?}", layouted.errors); + } + + #[cfg(not(debug_assertions))] { + println!(" - cold start: {:?}", warmup); + println!(" - warmed up: {:?}", total); + println!(" - parsing: {:?}", parse); + println!(" - layouting: {:?}", layout); + println!(); + } + + layouted.output } /// Command line options.