konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decoupled function parser 🔗 [WIP]

Browse Source
This commit is contained in:
Laurenz 2020-01-21 17:09:31 +01:00
parent 1c1c994c46
commit 78da2bdd5d
19 changed files with 627 additions and 638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -3,3 +3,4 @@
**/*.rs.bk
Cargo.lock
tests/cache
_things

2
src/bin/main.rs
View File

@ -41,7 +41,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap();
typesetter.add_font_provider(provider);
let layouts = block_on(typesetter.typeset(&src))?;
let layouts = block_on(typesetter.typeset(&src));
let exporter = PdfExporter::new();
let writer = BufWriter::new(File::create(&dest)?);

16
src/error.rs
View File

@ -1,10 +1,20 @@
#[derive(Debug, Clone, Eq, PartialEq)]
use serde::Serialize;
#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
pub struct Error {
pub message: String,
pub severity: Severity,
}
impl Error {
pub fn new(message: impl Into<String>) -> Error {
Error { message: message.into() }
pub fn new(message: impl Into<String>, severity: Severity) -> Error {
Error { message: message.into(), severity }
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize)]
pub enum Severity {
Warning,
Error,
}

22
src/func/macros.rs
View File

@ -22,7 +22,7 @@ macro_rules! function {
// Parse trait.
(@parse($($a:tt)*) parse(default) $($r:tt)*) => {
function!(@parse($($a)*) parse() { Default::default() } $($r)*);
function!(@parse($($a)*) parse(_h, _b, _c, _e, _d, _m) {Default::default() } $($r)*);
};
(@parse($($a:tt)*) parse($h:ident, $b:ident, $c:ident, $e:ident, $d:ident) $($r:tt)* ) => {
function!(@parse($($a)*) parse($h, $b, $c, $e, $d, _metadata) $($r)*);
@ -40,7 +40,7 @@ macro_rules! function {
fn parse(
#[allow(unused)] mut $header: FuncHeader,
#[allow(unused)] $body: Option<Spanned<&str>>,
#[allow(unused)] $body: Option<(Position, &str)>,
#[allow(unused)] $ctx: ParseContext,
#[allow(unused)] $metadata: Self::Meta,
) -> Parsed<Self> where Self: Sized {
@ -85,7 +85,7 @@ macro_rules! function {
macro_rules! body {
(opt: $body:expr, $ctx:expr, $errors:expr, $decos:expr) => ({
$body.map(|body| {
let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
let parsed = $crate::syntax::parse(body.0, body.1, $ctx);
$errors.extend(parsed.errors);
$decos.extend(parsed.decorations);
parsed.output
@ -99,13 +99,19 @@ macro_rules! body {
};
}
/// Construct a spanned error.
/// Construct an error with an optional span.
#[macro_export]
macro_rules! err {
($span:expr, $($args:tt)*) => {
$crate::syntax::Spanned {
v: $crate::error::Error::new(format!($($args)*)),
span: $span,
(@$severity:ident: $span:expr; $($args:tt)*) => {
$crate::syntax::Spanned { v: err!(@Error: $($args)*), span: $span }
};
(@$severity:ident: $($args:tt)*) => {
$crate::error::Error {
message: format!($($args)*),
severity: $crate::error::Severity::$severity,
}
};
($($tts:tt)*) => { err!(@Error: $($tts)*) };
}

13
src/func/mod.rs
View File

@ -1,6 +1,5 @@
//! Dynamic typesetting functions.
use std::any::Any;
use std::collections::HashMap;
use std::fmt::{self, Debug, Formatter};
@ -12,6 +11,7 @@ mod macros;
/// Useful imports for creating your own functions.
pub mod prelude {
pub use super::{Scope, Parse, Command, Commands};
pub use crate::error::Error;
pub use crate::layout::prelude::*;
pub use crate::syntax::prelude::*;
pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize};
@ -26,7 +26,7 @@ pub trait Parse {
/// Parse the header and body into this function given a context.
fn parse(
header: FuncHeader,
body: Option<Spanned<&str>>,
body: Option<(Position, &str)>,
ctx: ParseContext,
metadata: Self::Meta,
) -> Parsed<Self> where Self: Sized;
@ -36,7 +36,7 @@ pub trait Parse {
/// implements [`Model`].
type Parser = dyn Fn(
FuncHeader,
Option<Spanned<&str>>,
Option<(Position, &str)>,
ParseContext,
) -> Parsed<Box<dyn Model>>;
@ -102,11 +102,16 @@ impl Scope {
}
/// Return the parser with the given name if there is one.
pub(crate) fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
self.parsers.get(name)
.map(|x| &**x)
.ok_or_else(|| &*self.fallback)
}
/// Return the fallback parser.
pub fn get_fallback_parser(&self) -> &Parser {
&*self.fallback
}
}
impl Debug for Scope {

94
src/layout/mod.rs
View File

@ -5,7 +5,7 @@ use smallvec::SmallVec;
use toddle::query::{SharedFontLoader, FontIndex};
use crate::error::Error;
use crate::syntax::SpanVec;
use crate::syntax::{SyntaxModel, SpanVec};
use crate::size::{Size, Size2D, SizeBox};
use crate::style::LayoutStyle;
@ -26,7 +26,7 @@ pub mod prelude {
/// Different kinds of layouters (fully re-exported).
pub mod layouters {
pub use super::model::layout;
pub use super::model::ModelLayouter;
pub use super::line::{LineLayouter, LineContext};
pub use super::stack::{StackLayouter, StackContext};
pub use super::text::{layout_text, TextContext};
@ -37,20 +37,6 @@ pub use self::layouters::*;
pub use self::prelude::*;
pub struct Layouted<T> {
pub output: T,
pub errors: SpanVec<Error>,
}
impl<T> Layouted<T> {
pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
Layouted {
output: f(self.output),
errors: self.errors,
}
}
}
/// A collection of layouts.
pub type MultiLayout = Vec<Layout>;
@ -80,34 +66,6 @@ impl Layout {
}
}
/// Layout components that can be serialized.
pub trait Serialize {
/// Serialize the data structure into an output writable.
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
}
impl Serialize for Layout {
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
writeln!(f, "{}", self.actions.len())?;
for action in &self.actions {
action.serialize(f)?;
writeln!(f)?;
}
Ok(())
}
}
impl Serialize for MultiLayout {
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
writeln!(f, "{}", self.len())?;
for layout in self {
layout.serialize(f)?;
}
Ok(())
}
}
/// The general context for layouting.
#[derive(Debug, Clone)]
pub struct LayoutContext<'a, 'p> {
@ -133,6 +91,26 @@ pub struct LayoutContext<'a, 'p> {
pub debug: bool,
}
pub struct Layouted<T> {
pub output: T,
pub errors: SpanVec<Error>,
}
impl<T> Layouted<T> {
pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
Layouted {
output: f(self.output),
errors: self.errors,
}
}
}
pub async fn layout(model: &SyntaxModel, ctx: LayoutContext<'_, '_>) -> Layouted<MultiLayout> {
let mut layouter = ModelLayouter::new(ctx);
layouter.layout_syntax_model(model).await;
layouter.finish()
}
/// A possibly stack-allocated vector of layout spaces.
pub type LayoutSpaces = SmallVec<[LayoutSpace; 2]>;
@ -397,3 +375,31 @@ impl LastSpacing {
}
}
}
/// Layout components that can be serialized.
pub trait Serialize {
/// Serialize the data structure into an output writable.
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
}
impl Serialize for Layout {
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
writeln!(f, "{}", self.actions.len())?;
for action in &self.actions {
action.serialize(f)?;
writeln!(f)?;
}
Ok(())
}
}
impl Serialize for MultiLayout {
fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
writeln!(f, "{}", self.len())?;
for layout in self {
layout.serialize(f)?;
}
Ok(())
}
}

159
src/layout/model.rs
View File

@ -1,5 +1,3 @@
use std::pin::Pin;
use std::future::Future;
use smallvec::smallvec;
use crate::error::Error;
@ -9,17 +7,8 @@ use crate::syntax::{SpanVec, Spanned, Span, offset_spans};
use super::*;
pub async fn layout(
model: &SyntaxModel,
ctx: LayoutContext<'_, '_>
) -> Layouted<MultiLayout> {
let mut layouter = ModelLayouter::new(ctx);
layouter.layout_syntax_model(model).await;
layouter.finish()
}
#[derive(Debug, Clone)]
struct ModelLayouter<'a, 'p> {
pub struct ModelLayouter<'a, 'p> {
ctx: LayoutContext<'a, 'p>,
layouter: LineLayouter,
style: LayoutStyle,
@ -28,7 +17,7 @@ struct ModelLayouter<'a, 'p> {
impl<'a, 'p> ModelLayouter<'a, 'p> {
/// Create a new syntax tree layouter.
fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
pub fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
ModelLayouter {
layouter: LineLayouter::new(LineContext {
spaces: ctx.spaces.clone(),
@ -44,7 +33,7 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
}
}
fn layout<'r>(
pub fn layout<'r>(
&'r mut self,
model: Spanned<&'r dyn Model>
) -> DynFuture<'r, ()> { Box::pin(async move {
@ -60,75 +49,11 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
self.errors.extend(offset_spans(layouted.errors, model.span.start));
for command in commands {
self.execute_command(command, model.span);
self.execute_command(command, model.span).await;
}
}) }
fn execute_command<'r>(
&'r mut self,
command: Command<'r>,
model_span: Span,
) -> DynFuture<'r, ()> { Box::pin(async move {
use Command::*;
match command {
LayoutSyntaxModel(model) => self.layout_syntax_model(model).await,
Add(layout) => self.layouter.add(layout),
AddMultiple(layouts) => self.layouter.add_multiple(layouts),
SpacingFunc(space, kind, axis) => match axis {
Primary => self.layouter.add_primary_spacing(space, kind),
Secondary => self.layouter.add_secondary_spacing(space, kind),
}
FinishLine => self.layouter.finish_line(),
FinishSpace => self.layouter.finish_space(true),
BreakParagraph => self.layout_paragraph(),
BreakPage => {
if self.ctx.nested {
self.errors.push(Spanned::new(
Error::new( "page break cannot be issued from nested context"),
model_span,
));
} else {
self.layouter.finish_space(true)
}
}
SetTextStyle(style) => {
self.layouter.set_line_spacing(style.line_spacing());
self.style.text = style;
}
SetPageStyle(style) => {
if self.ctx.nested {
self.errors.push(Spanned::new(
Error::new("page style cannot be changed from nested context"),
model_span,
));
} else {
self.style.page = style;
let margins = style.margins();
self.ctx.base = style.dimensions.unpadded(margins);
self.layouter.set_spaces(smallvec![
LayoutSpace {
dimensions: style.dimensions,
padding: margins,
expansion: LayoutExpansion::new(true, true),
}
], true);
}
}
SetAlignment(alignment) => self.ctx.alignment = alignment,
SetAxes(axes) => {
self.layouter.set_axes(axes);
self.ctx.axes = axes;
}
}
}) }
fn layout_syntax_model<'r>(
pub fn layout_syntax_model<'r>(
&'r mut self,
model: &'r SyntaxModel
) -> DynFuture<'r, ()> { Box::pin(async move {
@ -161,6 +86,73 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
}
}) }
pub fn finish(self) -> Layouted<MultiLayout> {
Layouted {
output: self.layouter.finish(),
errors: self.errors,
}
}
fn execute_command<'r>(
&'r mut self,
command: Command<'r>,
span: Span,
) -> DynFuture<'r, ()> { Box::pin(async move {
use Command::*;
match command {
LayoutSyntaxModel(model) => self.layout_syntax_model(model).await,
Add(layout) => self.layouter.add(layout),
AddMultiple(layouts) => self.layouter.add_multiple(layouts),
SpacingFunc(space, kind, axis) => match axis {
Primary => self.layouter.add_primary_spacing(space, kind),
Secondary => self.layouter.add_secondary_spacing(space, kind),
}
FinishLine => self.layouter.finish_line(),
FinishSpace => self.layouter.finish_space(true),
BreakParagraph => self.layout_paragraph(),
BreakPage => {
if self.ctx.nested {
self.errors.push(err!(span;
"page break cannot be issued from nested context"));
} else {
self.layouter.finish_space(true)
}
}
SetTextStyle(style) => {
self.layouter.set_line_spacing(style.line_spacing());
self.style.text = style;
}
SetPageStyle(style) => {
if self.ctx.nested {
self.errors.push(err!(span;
"page style cannot be changed from nested context"));
} else {
self.style.page = style;
let margins = style.margins();
self.ctx.base = style.dimensions.unpadded(margins);
self.layouter.set_spaces(smallvec![
LayoutSpace {
dimensions: style.dimensions,
padding: margins,
expansion: LayoutExpansion::new(true, true),
}
], true);
}
}
SetAlignment(alignment) => self.ctx.alignment = alignment,
SetAxes(axes) => {
self.layouter.set_axes(axes);
self.ctx.axes = axes;
}
}
}) }
async fn layout_text(&mut self, text: &str) {
self.layouter.add(layout_text(text, TextContext {
loader: &self.ctx.loader,
@ -183,11 +175,4 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
SpacingKind::PARAGRAPH,
);
}
fn finish(self) -> Layouted<MultiLayout> {
Layouted {
output: self.layouter.finish(),
errors: self.errors,
}
}
}

3
src/lib.rs
View File

@ -24,10 +24,9 @@ use std::cell::RefCell;
use smallvec::smallvec;
use toddle::query::{FontLoader, FontProvider, SharedFontLoader};
use toddle::Error as FontError;
use crate::func::Scope;
use crate::layout::{Layouted, LayoutContext, MultiLayout};
use crate::layout::{Layouted, MultiLayout};
use crate::syntax::{parse, ParseContext, Parsed, SyntaxModel, Position};
use crate::style::{LayoutStyle, PageStyle, TextStyle};

8
src/library/maps/mod.rs
View File

@ -21,8 +21,8 @@ macro_rules! key {
impl ExpressionKind for $type {
const NAME: &'static str = $name;
fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
if let Expression::Ident(ident) = expr.v {
fn from_expr(expr: Spanned<Expr>) -> ParseResult<Self> {
if let Expr::Ident(ident) = expr.v {
Self::from_ident(&Spanned::new(ident, expr.span))
} else {
error!("expected {}", Self::NAME);
@ -55,8 +55,8 @@ impl<T> Into<Option<T>> for DefaultKey<T> {
impl<T> ExpressionKind for DefaultKey<T> where T: ExpressionKind {
const NAME: &'static str = T::NAME;
fn from_expr(expr: Spanned<Expression>) -> ParseResult<DefaultKey<T>> {
if let Expression::Ident(ident) = &expr.v {
fn from_expr(expr: Spanned<Expr>) -> ParseResult<DefaultKey<T>> {
if let Expr::Ident(ident) = &expr.v {
match ident.as_str() {
"default" => return Ok(DefaultKey::None),
_ => {},

12
src/library/mod.rs
View File

@ -63,8 +63,8 @@ function! {
body: parse!(optional: body, ctx),
list: {
header.args.iter_pos().map(|arg| match arg.v {
Expression::Str(s) |
Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
Expr::Str(s) |
Expr::Ident(Ident(s)) => Ok(s.to_lowercase()),
_ => error!("expected identifier or string"),
}).collect::<LayoutResult<Vec<_>>>()?
}
@ -117,8 +117,8 @@ function! {
parse(header, body, ctx) {
FontWeightFunc {
body: parse!(optional: body, ctx),
weight: match header.args.get_pos::<Expression>()? {
Expression::Number(weight) => {
weight: match header.args.get_pos::<Expr>()? {
Expr::Number(weight) => {
let weight = weight.round() as i16;
FontWeight(
if weight < 100 { 100 }
@ -126,7 +126,7 @@ function! {
else { 900 }
)
}
Expression::Ident(Ident(s)) => {
Expr::Ident(Ident(s)) => {
match FontWeight::from_str(&s) {
Some(weight) => weight,
None => error!("invalid font weight: `{}`", s),
@ -263,7 +263,7 @@ function! {
SpacingFunc {
axis: AxisKey::Specific(axis),
spacing: FSize::from_expr(
header.args.get_pos::<Spanned<Expression>>()?
header.args.get_pos::<Spanned<Expr>>()?
)?,
}
} else {

105
src/syntax/expr.rs
View File

@ -4,7 +4,7 @@ use super::*;
/// An argument or return value.
#[derive(Clone, PartialEq)]
pub enum Expression {
pub enum Expr {
Ident(Ident),
Str(String),
Number(f64),
@ -14,9 +14,24 @@ pub enum Expression {
Object(Object),
}
impl Display for Expression {
impl Expr {
pub fn name(&self) -> &'static str {
use Expr::*;
match self {
Ident(_) => "identifier",
Str(_) => "string",
Number(_) => "number",
Size(_) => "size",
Bool(_) => "boolean",
Tuple(_) => "tuple",
Object(_) => "object",
}
}
}
impl Display for Expr {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
use Expression::*;
use Expr::*;
match self {
Ident(i) => write!(f, "{}", i),
Str(s) => write!(f, "{:?}", s),
@ -29,6 +44,8 @@ impl Display for Expression {
}
}
debug_display!(Expr);
/// An identifier.
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Ident(pub String);
@ -53,10 +70,15 @@ impl Display for Ident {
}
}
debug_display!(Ident);
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct StringLike(pub String);
/// A sequence of expressions.
#[derive(Clone, PartialEq)]
pub struct Tuple {
pub items: Vec<Spanned<Expression>>,
pub items: Vec<Spanned<Expr>>,
}
impl Tuple {
@ -64,7 +86,7 @@ impl Tuple {
Tuple { items: vec![] }
}
pub fn add(&mut self, item: Spanned<Expression>) {
pub fn add(&mut self, item: Spanned<Expr>) {
self.items.push(item);
}
}
@ -86,6 +108,8 @@ impl Display for Tuple {
}
}
debug_display!(Tuple);
/// A key-value collection of identifiers and associated expressions.
#[derive(Clone, PartialEq)]
pub struct Object {
@ -97,7 +121,7 @@ impl Object {
Object { pairs: vec![] }
}
pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.pairs.push(Pair { key, value });
}
@ -127,11 +151,13 @@ impl Display for Object {
}
}
debug_display!(Object);
/// A key-value pair in an object.
#[derive(Clone, PartialEq)]
pub struct Pair {
pub key: Spanned<Ident>,
pub value: Spanned<Expression>,
pub value: Spanned<Expr>,
}
impl Display for Pair {
@ -140,57 +166,56 @@ impl Display for Pair {
}
}
debug_display!(Ident);
debug_display!(Expression);
debug_display!(Tuple);
debug_display!(Object);
debug_display!(Pair);
/// Kinds of expressions.
pub trait ExpressionKind: Sized {
pub trait ExprKind: Sized {
/// The name of the expression in an `expected <name>` error.
const NAME: &'static str;
/// Create from expression.
fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>;
fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error>;
}
impl<T> ExprKind for Spanned<T> where T: ExprKind {
const NAME: &'static str = T::NAME;
fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
let span = expr.span;
T::from_expr(expr).map(|v| Spanned { v, span })
}
}
/// Implements the expression kind trait for a type.
macro_rules! kind {
($type:ty, $name:expr, $($patterns:tt)*) => {
impl ExpressionKind for $type {
($type:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => {
impl ExprKind for $type {
const NAME: &'static str = $name;
fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
#[allow(unreachable_patterns)]
Ok(match expr.v {
$($patterns)*,
_ => error!("expected {}", Self::NAME),
$($p => $r),*,
_ => return Err(
err!("expected {}, found {}", Self::NAME, expr.v.name())
),
})
}
}
};
}
kind!(Expression, "expression", e => e);
kind!(Ident, "identifier", Expression::Ident(ident) => ident);
kind!(String, "string", Expression::Str(string) => string);
kind!(f64, "number", Expression::Number(num) => num);
kind!(bool, "boolean", Expression::Bool(boolean) => boolean);
kind!(Size, "size", Expression::Size(size) => size);
kind!(Tuple, "tuple", Expression::Tuple(tuple) => tuple);
kind!(Object, "object", Expression::Object(object) => object);
kind!(ScaleSize, "number or size",
Expression::Size(size) => ScaleSize::Absolute(size),
Expression::Number(scale) => ScaleSize::Scaled(scale as f32)
kind!(Expr, "expression", e => e);
kind!(Ident, "identifier", Expr::Ident(i) => i);
kind!(String, "string", Expr::Str(s) => s);
kind!(f64, "number", Expr::Number(n) => n);
kind!(bool, "boolean", Expr::Bool(b) => b);
kind!(Size, "size", Expr::Size(s) => s);
kind!(Tuple, "tuple", Expr::Tuple(t) => t);
kind!(Object, "object", Expr::Object(o) => o);
kind!(ScaleSize, "number or size",
Expr::Size(size) => ScaleSize::Absolute(size),
Expr::Number(scale) => ScaleSize::Scaled(scale as f32),
);
kind!(StringLike, "identifier or string",
Expr::Ident(Ident(s)) => StringLike(s),
Expr::Str(s) => StringLike(s),
);
impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind {
const NAME: &'static str = T::NAME;
fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> {
let span = expr.span;
T::from_expr(expr).map(|v| Spanned { v, span })
}
}

8
src/syntax/func.rs
View File

@ -15,7 +15,7 @@ pub struct FuncArgs {
#[derive(Debug, Clone, PartialEq)]
pub enum Arg {
Pos(Spanned<Expression>),
Pos(Spanned<Expr>),
Key(Pair),
}
@ -46,12 +46,12 @@ impl FuncArgs {
}
/// Add a positional argument.
pub fn add_pos(&mut self, item: Spanned<Expression>) {
pub fn add_pos(&mut self, item: Spanned<Expr>) {
self.pos.add(item);
}
/// Add a keyword argument.
pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
self.key.add(key, value);
}
@ -92,7 +92,7 @@ impl FuncArgs {
// }
// /// Iterator over positional arguments.
// pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
// pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expr>> {
// let tuple = std::mem::replace(&mut self.positional, Tuple::new());
// tuple.items.into_iter()
// }

21
src/syntax/mod.rs
View File

@ -23,22 +23,6 @@ pub mod prelude {
}
pub struct Parsed<T> {
pub output: T,
pub errors: SpanVec<Error>,
pub decorations: SpanVec<Decoration>,
}
impl<T> Parsed<T> {
pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
Parsed {
output: f(self.output),
errors: self.errors,
decorations: self.decorations,
}
}
}
#[async_trait::async_trait(?Send)]
pub trait Model: Debug + ModelBounds {
async fn layout<'a>(
@ -110,7 +94,7 @@ impl SyntaxModel {
#[async_trait::async_trait(?Send)]
impl Model for SyntaxModel {
async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
Layouted {
output: vec![Command::LayoutSyntaxModel(self)],
errors: vec![],
@ -153,7 +137,8 @@ impl PartialEq for Node {
}
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum Decoration {
ValidFuncName,
InvalidFuncName,

394
src/syntax/parsing.rs
View File

@ -3,10 +3,6 @@ use super::*;
use Token::*;
pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
Parser::new(start, src, ctx).parse()
}
/// The context for parsing.
#[derive(Debug, Copy, Clone)]
pub struct ParseContext<'a> {
@ -14,185 +10,169 @@ pub struct ParseContext<'a> {
pub scope: &'a Scope,
}
struct Parser<'s> {
src: &'s str,
ctx: ParseContext<'s>,
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
position: Position,
last_position: Position,
errors: SpanVec<Error>,
decorations: SpanVec<Decoration>,
pub struct Parsed<T> {
pub output: T,
pub errors: SpanVec<Error>,
pub decorations: SpanVec<Decoration>,
}
impl<'s> Parser<'s> {
fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
Parser {
src,
impl<T> Parsed<T> {
pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
Parsed {
output: f(self.output),
errors: self.errors,
decorations: self.decorations,
}
}
}
pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
let mut model = SyntaxModel::new();
let mut errors = Vec::new();
let mut decorations = Vec::new();
let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
while let Some(token) = tokens.next() {
let span = token.span;
let node = match token.v {
Space(newlines) => if newlines >= 2 {
Node::Newline
} else {
Node::Space
},
Function { header, body, terminated } => {
let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
errors.extend(offset_spans(parsed.errors, span.start));
decorations.extend(offset_spans(parsed.decorations, span.start));
if !terminated {
errors.push(err!(Span::at(span.end); "expected closing bracket"));
}
parsed.output
}
Star => Node::ToggleBolder,
Underscore => Node::ToggleItalic,
Backtick => Node::ToggleMonospace,
Text(text) => Node::Text(text.to_owned()),
LineComment(_) | BlockComment(_) => continue,
other => {
errors.push(err!(span; "unexpected {}", name(other)));
continue;
}
};
model.add(Spanned { v: node, span: token.span });
}
Parsed { output: model, errors, decorations }
}
struct FuncParser<'s> {
ctx: ParseContext<'s>,
errors: SpanVec<Error>,
decorations: SpanVec<Decoration>,
tokens: Tokens<'s>,
peeked: Option<Option<Spanned<Token<'s>>>>,
body: Option<(Position, &'s str)>,
}
impl<'s> FuncParser<'s> {
fn new(
header: &'s str,
body: Option<(Position, &'s str)>,
ctx: ParseContext<'s>
) -> FuncParser<'s> {
FuncParser {
ctx,
tokens: tokenize(start, src),
peeked: None,
position: Position::ZERO,
last_position: Position::ZERO,
errors: vec![],
decorations: vec![],
tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
peeked: None,
body,
}
}
/// The main parsing entrypoint.
fn parse(mut self) -> Parsed<SyntaxModel> {
let mut model = SyntaxModel::new();
while let Some(token) = self.eat() {
let mut span = token.span;
let node = match token.v {
LineComment(_) | BlockComment(_) => None,
Whitespace(newlines) => Some(if newlines >= 2 {
Node::Newline
} else {
Node::Space
}),
LeftBracket => self.parse_func().map(|spanned| {
span = spanned.span;
spanned.v
}),
Star => Some(Node::ToggleBolder),
Underscore => Some(Node::ToggleItalic),
Backtick => Some(Node::ToggleMonospace),
Text(text) => Some(Node::Text(text.to_owned())),
_ => {
self.unexpected(token);
None
fn parse(mut self) -> Parsed<Node> {
let parsed = if let Some(header) = self.parse_func_header() {
let name = header.name.v.as_str();
let (parser, deco) = match self.ctx.scope.get_parser(name) {
Ok(parser) => (parser, Decoration::ValidFuncName),
Err(parser) => {
self.errors.push(err!(header.name.span; "unknown function"));
(parser, Decoration::InvalidFuncName)
}
};
if let Some(v) = node {
model.add(Spanned { v, span });
}
}
self.decorations.push(Spanned::new(deco, header.name.span));
parser(header, self.body, self.ctx)
} else {
let default = FuncHeader {
name: Spanned::new(Ident("".to_string()), Span::ZERO),
args: FuncArgs::new(),
};
// Use the fallback function such that the body is still rendered
// even if the header is completely unparsable.
self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
};
self.errors.extend(parsed.errors);
self.decorations.extend(parsed.decorations);
Parsed {
output: model,
output: Node::Model(parsed.output),
errors: self.errors,
decorations: self.decorations,
}
}
/// Parses a function including header and body with the cursor starting
/// right behind the first opening bracket.
fn parse_func(&mut self) -> Option<Spanned<Node>> {
let start = self.last_pos();
let header = self.parse_func_header();
self.eat_until(|t| t == RightBracket, false);
if self.eat().map(Spanned::value) != Some(RightBracket) {
self.expected_at("closing bracket", self.pos());
}
let body = if self.peekv() == Some(LeftBracket) {
self.eat();
let start_index = self.tokens.index();
let start_position = self.tokens.pos();
let found = self.tokens.move_to_closing_bracket();
let end_index = self.tokens.index();
let end_position = self.tokens.pos();
let body = &self.src[start_index .. end_index];
self.position = end_position;
if found {
let next = self.eat().map(Spanned::value);
debug_assert_eq!(next, Some(RightBracket));
} else {
self.expected_at("closing bracket", self.pos());
}
Some(Spanned::new(body, Span::new(start_position, end_position)))
} else {
None
};
let header = header?;
let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
Ok(parser) => (parser, Decoration::ValidFuncName),
Err(parser) => {
let error = Error::new(format!("unknown function: `{}`", header.name.v));
self.errors.push(Spanned::new(error, header.name.span));
(parser, Decoration::InvalidFuncName)
}
};
self.decorations.push(Spanned::new(decoration, header.name.span));
let parsed = parser(header, body, self.ctx);
self.errors.extend(offset_spans(parsed.errors, start));
self.decorations.extend(offset_spans(parsed.decorations, start));
let node = Node::Model(parsed.output);
let end = self.pos();
let span = Span { start, end };
Some(Spanned { v: node, span })
}
/// Parses a function header including the closing bracket.
fn parse_func_header(&mut self) -> Option<FuncHeader> {
let start = self.pos();
self.skip_whitespace();
let name = self.parse_func_name()?;
let name = match self.eat() {
Some(Spanned { v: ExprIdent(ident), span }) => {
Spanned { v: Ident(ident.to_string()), span }
}
other => {
self.expected_found_or_at("identifier", other, start);
return None;
}
};
self.skip_whitespace();
let args = match self.peek() {
Some(Spanned { v: Colon, .. }) => {
self.eat();
self.parse_func_args()
}
Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
other => {
self.expected_at("colon or closing bracket", name.span.end);
let args = match self.eat().map(Spanned::value) {
Some(Colon) => self.parse_func_args(),
Some(_) => {
self.expected_at("colon", name.span.end);
FuncArgs::new()
}
None => FuncArgs::new(),
};
Some(FuncHeader { name, args })
}
/// Parses the function name if is the next token. Otherwise, it adds an
/// error and returns `None`.
fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
match self.peek() {
Some(Spanned { v: ExprIdent(ident), span }) => {
self.eat();
return Some(Spanned { v: Ident(ident.to_string()), span });
}
other => self.expected_found_or_at("identifier", other, self.pos()),
}
None
}
/// Parses the function arguments and stops right before the final closing
/// bracket.
fn parse_func_args(&mut self) -> FuncArgs {
let mut args = FuncArgs::new();
loop {
self.skip_whitespace();
match self.peekv() {
Some(RightBracket) | None => break,
_ => match self.parse_arg() {
Some(arg) => args.add(arg),
None => {}
}
self.skip_whitespace();
while self.peek().is_some() {
match self.parse_arg() {
Some(arg) => args.add(arg),
None => {}
}
self.skip_whitespace();
}
args
@ -221,7 +201,7 @@ impl<'s> Parser<'s> {
})
})
} else {
Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span)))
}
} else {
self.parse_expr().map(|expr| Arg::Pos(expr))
@ -230,7 +210,6 @@ impl<'s> Parser<'s> {
if let Some(arg) = &arg {
self.skip_whitespace();
match self.peekv() {
Some(RightBracket) => {}
Some(Comma) => { self.eat(); }
Some(_) => self.expected_at("comma", arg.span().end),
_ => {}
@ -244,19 +223,27 @@ impl<'s> Parser<'s> {
}
/// Parse a atomic or compound (tuple / object) expression.
fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
let first = self.peek()?;
let mut expr = |v| {
self.eat();
Spanned { v, span: first.span }
};
let spanned = |v| Spanned { v, span: first.span };
Some(match first.v {
ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
ExprStr(s) => expr(Expression::Str(s.to_string())),
ExprNumber(n) => expr(Expression::Number(n)),
ExprSize(s) => expr(Expression::Size(s)),
ExprBool(b) => expr(Expression::Bool(b)),
ExprIdent(i) => {
self.eat();
spanned(Expr::Ident(Ident(i.to_string())))
}
ExprStr { string, terminated } => {
if !terminated {
self.expected_at("quote", first.span.end);
}
self.eat();
spanned(Expr::Str(string.to_string()))
}
ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) }
ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) }
ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) }
LeftParen => self.parse_tuple(),
LeftBrace => self.parse_object(),
_ => return None,
@ -264,56 +251,48 @@ impl<'s> Parser<'s> {
}
/// Parse a tuple expression.
fn parse_tuple(&mut self) -> Spanned<Expression> {
fn parse_tuple(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
if self.peekv() == Some(RightParen) {
self.eat();
}
self.eat_until(|t| t == RightParen, true);
let end = self.pos();
let span = Span { start, end };
Spanned { v: Expression::Tuple(Tuple::new()), span }
Spanned { v: Expr::Tuple(Tuple::new()), span }
}
/// Parse an object expression.
fn parse_object(&mut self) -> Spanned<Expression> {
fn parse_object(&mut self) -> Spanned<Expr> {
let start = self.pos();
// TODO: Do the thing.
self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
if self.peekv() == Some(RightBrace) {
self.eat();
}
self.eat_until(|t| t == RightBrace, true);
let end = self.pos();
let span = Span { start, end };
Spanned { v: Expression::Object(Object::new()), span }
Spanned { v: Expr::Object(Object::new()), span }
}
/// Skip all whitespace/comment tokens.
fn skip_whitespace(&mut self) {
self.eat_until(|t|
!matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
}
/// Add an error about an `thing` which was expected but not found at the
/// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
let error = Error::new(format!("expected {}", thing));
self.errors.push(Spanned::new(error, Span::at(pos)));
!matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false)
}
/// Add an error about an expected `thing` which was not found, showing
/// what was found instead.
fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
let message = format!("expected {}, found {}", thing, name(found.v));
let error = Error::new(message);
self.errors.push(Spanned::new(error, found.span));
self.errors.push(err!(found.span;
"expected {}, found {}", thing, name(found.v)));
}
/// Add an error about an `thing` which was expected but not found at the
/// given position.
fn expected_at(&mut self, thing: &str, pos: Position) {
self.errors.push(err!(Span::at(pos); "expected {}", thing));
}
/// Add a found-error if `found` is some and a positional error, otherwise.
@ -329,12 +308,6 @@ impl<'s> Parser<'s> {
}
}
/// Add an error about an unexpected token `found`.
fn unexpected(&mut self, found: Spanned<Token>) {
let error = Error::new(format!("unexpected {}", name(found.v)));
self.errors.push(Spanned::new(error, found.span));
}
/// Consume tokens until the function returns true and only consume the last
/// token if instructed to.
fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
@ -351,19 +324,10 @@ impl<'s> Parser<'s> {
}
}
/// Consume and return the next token, update positions and colorize the
/// token. All colorable tokens are per default colorized here, to override
/// a colorization use `Colorization::replace_last`.
/// Consume and return the next token.
fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
let token = self.peeked.take()
.unwrap_or_else(|| self.tokens.next());
if let Some(token) = token {
self.last_position = self.position;
self.position = token.span.end;
}
token
self.peeked.take()
.unwrap_or_else(|| self.tokens.next())
}
/// Peek at the next token without consuming it.
@ -379,23 +343,18 @@ impl<'s> Parser<'s> {
/// The position at the end of the last eat token / start of the peekable
/// token.
fn pos(&self) -> Position {
self.position
}
/// The position at the start of the last eaten token.
fn last_pos(&self) -> Position {
self.last_position
self.peeked.flatten()
.map(|s| s.span.start)
.unwrap_or_else(|| self.tokens.pos())
}
}
/// The name of a token in an `expected <...>` error.
/// The name of a token in an `(un)expected <...>` error.
fn name(token: Token) -> &'static str {
match token {
Whitespace(_) => "whitespace",
Space(_) => "space",
LineComment(_) | BlockComment(_) => "comment",
StarSlash => "end of block comment",
LeftBracket => "opening bracket",
RightBracket => "closing bracket",
Function { .. } => "function",
LeftParen => "opening paren",
RightParen => "closing paren",
LeftBrace => "opening brace",
@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str {
Comma => "comma",
Equals => "equals sign",
ExprIdent(_) => "identifier",
ExprStr(_) => "string",
ExprStr { .. } => "string",
ExprNumber(_) => "number",
ExprSize(_) => "size",
ExprBool(_) => "bool",
ExprBool(_) => "boolean",
Star => "star",
Underscore => "underscore",
Backtick => "backtick",
Text(_) => "invalid identifier",
Invalid("]") => "closing bracket",
Invalid("*/") => "end of block comment",
Invalid(_) => "invalid token",
}
}

46
src/syntax/span.rs
View File

@ -1,7 +1,7 @@
//! Spans map elements to the part of source code they originate from.
use std::fmt::{self, Debug, Display, Formatter};
use std::ops::{Add, AddAssign};
use std::ops::{Add, Sub};
use serde::Serialize;
@ -21,12 +21,13 @@ impl<T> Spanned<T> {
self.v
}
pub fn map<F, V>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
Spanned { v: f(self.v), span: self.span }
}
pub fn map_v<V>(&self, new_v: V) -> Spanned<V> {
Spanned { v: new_v, span: self.span }
pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
self.span = f(self.span);
self
}
}
@ -74,6 +75,13 @@ impl Span {
pub fn expand(&mut self, other: Span) {
*self = Span::merge(*self, other)
}
pub fn offset(self, start: Position) -> Span {
Span {
start: start + self.start,
end: start + self.end,
}
}
}
impl Display for Span {
@ -119,9 +127,21 @@ impl Add for Position {
}
}
impl AddAssign for Position {
fn add_assign(&mut self, other: Position) {
*self = *self + other;
impl Sub for Position {
type Output = Position;
fn sub(self, rhs: Position) -> Position {
if self.line == rhs.line {
Position {
line: 0,
column: self.column - rhs.column
}
} else {
Position {
line: self.line - rhs.line,
column: self.column,
}
}
}
}
@ -136,14 +156,6 @@ debug_display!(Position);
/// A vector of spanned things.
pub type SpanVec<T> = Vec<Spanned<T>>;
pub fn offset_spans<T>(
vec: SpanVec<T>,
start: Position,
) -> impl Iterator<Item=Spanned<T>> {
vec.into_iter()
.map(move |mut spanned| {
spanned.span.start += start;
spanned.span.end += start;
spanned
})
pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> {
vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
}

180
src/syntax/tokens.rs
View File

@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID;
use super::*;
use Token::*;
use State::*;
use TokenizationMode::*;
/// A minimal semantic entity of source code.
@ -12,20 +12,20 @@ use State::*;
pub enum Token<'s> {
/// One or more whitespace characters. The contained `usize` denotes the
/// number of newlines that were contained in the whitespace.
Whitespace(usize),
Space(usize),
/// A line comment with inner string contents `//<&'s str>\n`.
LineComment(&'s str),
/// A block comment with inner string contents `/*<&'s str>*/`. The comment
/// can contain nested block comments.
BlockComment(&'s str),
/// An erroneous `*/` without an opening block comment.
StarSlash,
/// A left bracket: `[`.
LeftBracket,
/// A right bracket: `]`.
RightBracket,
/// A function invocation `[<header>][<body>]`.
Function {
header: &'s str,
body: Option<(Position, &'s str)>,
terminated: bool,
},
/// A left parenthesis in a function header: `(`.
LeftParen,
@ -46,7 +46,7 @@ pub enum Token<'s> {
/// An identifier in a function header: `center`.
ExprIdent(&'s str),
/// A quoted string in a function header: `"..."`.
ExprStr(&'s str),
ExprStr { string: &'s str, terminated: bool },
/// A number in a function header: `3.14`.
ExprNumber(f64),
/// A size in a function header: `12pt`.
@ -63,36 +63,31 @@ pub enum Token<'s> {
/// Any other consecutive string.
Text(&'s str),
}
/// Decomposes text into a sequence of semantic tokens.
pub fn tokenize(start: Position, src: &str) -> Tokens {
Tokens::new(start, src)
/// Things that are not valid in the context they appeared in.
Invalid(&'s str),
}
/// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> {
src: &'s str,
state: State,
stack: Vec<(State, Position)>,
mode: TokenizationMode,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
enum State {
pub enum TokenizationMode {
Header,
StartBody,
Body,
}
impl<'s> Tokens<'s> {
pub fn new(start: Position, src: &'s str) -> Tokens<'s> {
pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
Tokens {
src,
state: State::Body,
stack: vec![],
mode,
iter: src.chars().peekable(),
position: start,
index: 0,
@ -110,35 +105,6 @@ impl<'s> Tokens<'s> {
pub fn pos(&self) -> Position {
self.position
}
/// Move through the string until an unbalanced closing bracket is found
/// without tokenizing the contents.
///
/// Returns whether a closing bracket was found or the end of the string was
/// reached.
pub fn move_to_closing_bracket(&mut self) -> bool {
let mut escaped = false;
let mut depth = 0;
self.read_string_until(|n| {
match n {
'[' if !escaped => depth += 1,
']' if !escaped => {
if depth == 0 {
return true;
} else {
depth -= 1;
}
}
'\\' => escaped = !escaped,
_ => escaped = false,
}
false
}, false, 0, 0);
self.peek() == Some(']')
}
}
impl<'s> Iterator for Tokens<'s> {
@ -153,55 +119,31 @@ impl<'s> Iterator for Tokens<'s> {
// Comments.
'/' if self.peek() == Some('/') => self.parse_line_comment(),
'/' if self.peek() == Some('*') => self.parse_block_comment(),
'*' if self.peek() == Some('/') => { self.eat(); StarSlash }
'*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace.
c if c.is_whitespace() => self.parse_whitespace(start),
// Functions.
'[' => {
match self.state {
Header | Body => {
self.stack.push((self.state, start));
self.position = Position::new(0, '['.len_utf8());
self.state = Header;
}
StartBody => self.state = Body,
}
LeftBracket
}
']' => {
if self.state == Header && self.peek() == Some('[') {
self.state = StartBody;
} else {
if let Some((state, pos)) = self.stack.pop() {
self.state = state;
self.position = pos + self.position;
} else {
self.state = Body;
}
}
RightBracket
}
'[' => self.parse_function(start),
']' => Invalid("]"),
// Syntactic elements in function headers.
'(' if self.state == Header => LeftParen,
')' if self.state == Header => RightParen,
'{' if self.state == Header => LeftBrace,
'}' if self.state == Header => RightBrace,
':' if self.state == Header => Colon,
',' if self.state == Header => Comma,
'=' if self.state == Header => Equals,
'(' if self.mode == Header => LeftParen,
')' if self.mode == Header => RightParen,
'{' if self.mode == Header => LeftBrace,
'}' if self.mode == Header => RightBrace,
':' if self.mode == Header => Colon,
',' if self.mode == Header => Comma,
'=' if self.mode == Header => Equals,
// String values.
'"' if self.state == Header => self.parse_string(),
'"' if self.mode == Header => self.parse_string(),
// Style toggles.
'*' if self.state == Body => Star,
'_' if self.state == Body => Underscore,
'`' if self.state == Body => Backtick,
'*' if self.mode == Body => Star,
'_' if self.mode == Body => Underscore,
'`' if self.mode == Body => Backtick,
// An escaped thing.
'\\' => self.parse_escaped(),
@ -215,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> {
',' | '"' | '/' => true,
_ => false,
}
}, false, -(c.len_utf8() as isize), 0);
}, false, -(c.len_utf8() as isize), 0).0;
if self.state == Header {
if self.mode == Header {
self.parse_expr(text)
} else {
Text(text)
@ -234,7 +176,7 @@ impl<'s> Iterator for Tokens<'s> {
impl<'s> Tokens<'s> {
fn parse_line_comment(&mut self) -> Token<'s> {
LineComment(self.read_string_until(is_newline_char, false, 1, 0))
LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
}
fn parse_block_comment(&mut self) -> Token<'s> {
@ -262,19 +204,60 @@ impl<'s> Tokens<'s> {
}
false
}, true, 0, -2))
}, true, 0, -2).0)
}
fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos();
Whitespace(end.line - start.line)
Space(end.line - start.line)
}
fn parse_function(&mut self, start: Position) -> Token<'s> {
let (header, terminated) = self.read_function_part();
self.eat();
if self.peek() != Some('[') {
return Function { header, body: None, terminated };
}
self.eat();
let offset = self.pos() - start;
let (body, terminated) = self.read_function_part();
self.eat();
Function { header, body: Some((offset, body)), terminated }
}
fn read_function_part(&mut self) -> (&'s str, bool) {
let mut escaped = false;
let mut in_string = false;
let mut depth = 0;
self.read_string_until(|n| {
match n {
'"' if !escaped => in_string = !in_string,
'[' if !escaped && !in_string => depth += 1,
']' if !escaped && !in_string => {
if depth == 0 {
return true;
} else {
depth -= 1;
}
}
'\\' => escaped = !escaped,
_ => escaped = false,
}
false
}, false, 0, 0)
}
fn parse_string(&mut self) -> Token<'s> {
let mut escaped = false;
ExprStr(self.read_string_until(|n| {
let (string, terminated) = self.read_string_until(|n| {
match n {
'"' if !escaped => return true,
'\\' => escaped = !escaped,
@ -282,7 +265,8 @@ impl<'s> Tokens<'s> {
}
false
}, true, 0, -1))
}, true, 0, -1);
ExprStr { string, terminated }
}
fn parse_escaped(&mut self) -> Token<'s> {
@ -294,7 +278,7 @@ impl<'s> Tokens<'s> {
}
let c = self.peek().unwrap_or('n');
if self.state == Body && is_escapable(c) {
if self.mode == Body && is_escapable(c) {
let index = self.index();
self.eat();
Text(&self.src[index .. index + c.len_utf8()])
@ -315,7 +299,7 @@ impl<'s> Tokens<'s> {
} else if is_identifier(text) {
ExprIdent(text)
} else {
Text(text)
Invalid(text)
}
}
@ -325,7 +309,7 @@ impl<'s> Tokens<'s> {
eat_match: bool,
offset_start: isize,
offset_end: isize,
) -> &'s str where F: FnMut(char) -> bool {
) -> (&'s str, bool) where F: FnMut(char) -> bool {
let start = ((self.index() as isize) + offset_start) as usize;
let mut matched = false;
@ -346,7 +330,7 @@ impl<'s> Tokens<'s> {
end = ((end as isize) + offset_end) as usize;
}
&self.src[start .. end]
(&self.src[start .. end], matched)
}
fn eat(&mut self) -> Option<char> {

1
tests/layouter/test.typ
View File

@ -1 +0,0 @@
[box][hi]

123
tests/parser/tokens.rs
View File

@ -1,80 +1,77 @@
// Whitespace.
t "" => []
t " " => [W(0)]
t " " => [W(0)]
t "\t" => [W(0)]
t " \t" => [W(0)]
t "\n" => [W(1)]
t "\n " => [W(1)]
t " \n" => [W(1)]
t " \n " => [W(1)]
t " \n\t \n " => [W(2)]
t "\r\n" => [W(1)]
t " \r\r\n \x0D" => [W(3)]
t "\n\r" => [W(2)]
t " " => [S(0)]
t " " => [S(0)]
t "\t" => [S(0)]
t " \t" => [S(0)]
t "\n" => [S(1)]
t "\n " => [S(1)]
t " \n" => [S(1)]
t " \n " => [S(1)]
t " \n\t \n " => [S(2)]
t "\r\n" => [S(1)]
t " \r\r\n \x0D" => [S(3)]
t "\n\r" => [S(2)]
// Comments.
t "a // bc\n " => [T("a"), W(0), LC(" bc"), W(1)]
t "a //a//b\n " => [T("a"), W(0), LC("a//b"), W(1)]
t "a //a//b\r\n" => [T("a"), W(0), LC("a//b"), W(1)]
t "a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")]
t "a // bc\n " => [T("a"), S(0), LC(" bc"), S(1)]
t "a //a//b\n " => [T("a"), S(0), LC("a//b"), S(1)]
t "a //a//b\r\n" => [T("a"), S(0), LC("a//b"), S(1)]
t "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]
t "/**/" => [BC("")]
t "_/*_/*a*/*/" => [U, BC("_/*a*/")]
t "_/*_/*a*/*/" => [Underscore, BC("_/*a*/")]
t "/*/*/" => [BC("/*/")]
t "abc*/" => [T("abc"), SS]
t "abc*/" => [T("abc"), Invalid("*/")]
// Header only tokens.
t "[" => [LB]
t "]" => [RB]
t "[(){}:=,]" => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB]
t "[a:b]" => [LB, ID("a"), CL, ID("b"), RB]
t "[🌓, 🌍,]" => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB]
t "[=]" => [LB, EQ, RB]
t "[,]" => [LB, CM, RB]
t "a: b" => [T("a"), T(":"), W(0), T("b")]
t "c=d, " => [T("c"), T("=d"), T(","), W(0)]
t r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB]
t r#"["hi", 12pt]"# => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB]
t "\"hi\"" => [T("\"hi"), T("\"")]
t "[a: true, x=1]" => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
ID("x"), EQ, NUM(1.0), RB]
t "[120%]" => [LB, NUM(1.2), RB]
th "[" => [Func("", None, false)]
th "]" => [Invalid("]")]
th "(){}:=," => [LP, RP, LB, RB, Colon, Equals, Comma]
th "a:b" => [Id("a"), Colon, Id("b")]
th "=" => [Equals]
th "," => [Comma]
th r#""hello\"world""# => [Str(r#"hello\"world"#)]
th r#""hi", 12pt"# => [Str("hi"), Comma, S(0), Size(12.0)]
th "\"hi\"" => [T("\"hi"), T("\"")]
th "a: true, x=1" => [Id("a"), Colon, S(0), Bool(true), Comma, S(0),
Id("x"), Equals, Num(1.0)]
th "120%" => [Num(1.2)]
th "🌓, 🌍," => [T("🌓"), Comma, S(0), T("🌍"), Comma]
tb "a: b" => [T("a"), T(":"), S(0), T("b")]
tb "c=d, " => [T("c"), T("=d"), T(","), S(0)]
// Body only tokens.
t "_*`" => [U, S, B]
t "[func]*bold*" => [LB, ID("func"), RB, S, T("bold"), S]
t "[_*`]" => [LB, T("_"), T("*"), T("`"), RB]
t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
tb "_*`" => [Underscore, Star, Backtick]
tb "[func]*bold*" => [Func("func", None, true), Star, T("bold"), Star]
tb "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]
th "_*`" => [Invalid("_"), Invalid("*"), Invalid("`")]
// Nested functions.
t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB]
t "[_][[,],]," => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")]
t "[=][=][=]" => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB]
t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB]
tb "[f: [=][*]]" => [Func("f: [=][*]", None, true)]
tb "[_][[,],]," => [Func("_", Some("[,],"), true), T(",")]
tb "[=][=][=]" => [Func("=", Some("="), true), Func("=", None, true)]
tb "[=][[=][=][=]]" => [Func("=", Some("[=][=][=]")), true]
// Escapes.
t r"\[" => [T("[")]
t r"\]" => [T("]")]
t r"\\" => [T(r"\")]
t r"\/" => [T("/")]
t r"\*" => [T("*")]
t r"\_" => [T("_")]
t r"\`" => [T("`")]
tb r"\[" => [T("[")]
tb r"\]" => [T("]")]
tb r"\\" => [T(r"\")]
tb r"\/" => [T("/")]
tb r"\*" => [T("*")]
tb r"\_" => [T("_")]
tb r"\`" => [T("`")]
// Unescapable special symbols.
t r"\:" => [T(r"\"), T(":")]
t r"\=" => [T(r"\"), T("=")]
t r"[\:]" => [LB, T(r"\"), CL, RB]
t r"[\=]" => [LB, T(r"\"), EQ, RB]
t r"[\,]" => [LB, T(r"\"), CM, RB]
th r"\:" => [T(r"\"), T(":")]
th r"\=" => [T(r"\"), T("=")]
th r"\:" => [T(r"\"), Colon]
th r"\=" => [T(r"\"), Equals]
th r"\," => [T(r"\"), Comma]
// Spans
ts "hello" => [(0:0, 0:5, T("hello"))]
ts "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, W(1)), (1:0, 1:1, T("c"))]
ts "[a=10]" => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ),
(0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)]
ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)),
(0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")),
(0:10, 0:11, RB), (0:11, 0:12, S)]
ts "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))]
ts "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)]
// Spans.
tbs "hello" => [(0:0, 0:5, T("hello"))]
tbs "ab\r\nc" => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]
tbs "[x = \"(1)\"]*" => [(0:0, 0:11, Func("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]
tbs "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]
tbs "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]
ths "a=10" => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]

57
tests/src/layouter.rs
View File

@ -10,8 +10,8 @@ use futures_executor::block_on;
use typstc::Typesetter;
use typstc::layout::{MultiLayout, Serialize};
use typstc::size::{Size, Size2D};
use typstc::style::PageStyle;
use typstc::size::{Size, Size2D, ValueBox};
use typstc::style::{PageStyle, PaperClass};
use typstc::toddle::query::FileSystemFontProvider;
use typstc::export::pdf::PdfExporter;
@ -62,14 +62,15 @@ fn main() -> DynResult<()> {
Ok(())
}
/// Create a _PDF_ with a name from the source code.
/// Create a _PDF_ and render with a name from the source code.
fn test(name: &str, src: &str) -> DynResult<()> {
println!("Testing: {}.", name);
let mut typesetter = Typesetter::new();
typesetter.set_page_style(PageStyle {
class: PaperClass::Custom,
dimensions: Size2D::with_all(Size::pt(250.0)),
.. PageStyle::default()
margins: ValueBox::with_all(None),
});
let provider = FileSystemFontProvider::from_index("../fonts/index.json")?;
@ -120,31 +121,43 @@ fn test(name: &str, src: &str) -> DynResult<()> {
/// Compile the source code with the typesetter.
fn compile(typesetter: &Typesetter, src: &str) -> MultiLayout {
#[cfg(not(debug_assertions))] {
use std::time::Instant;
#![allow(unused_variables)]
use std::time::Instant;
// Warmup.
// Warmup.
#[cfg(not(debug_assertions))]
let warmup = {
let warmup_start = Instant::now();
block_on(typesetter.typeset(&src));
let warmup_end = Instant::now();
Instant::now() - warmup_start
};
let start = Instant::now();
let tree = typesetter.parse(&src).output;
let mid = Instant::now();
let layouts = block_on(typesetter.layout(&tree)).output;
let end = Instant::now();
let start = Instant::now();
let parsed = typesetter.parse(&src);
let parse = Instant::now() - start;
println!(" - cold start: {:?}", warmup_end - warmup_start);
println!(" - warmed up: {:?}", end - start);
println!(" - parsing: {:?}", mid - start);
println!(" - layouting: {:?}", end - mid);
println!();
layouts
if !parsed.errors.is_empty() {
println!("parse errors: {:#?}", parsed.errors);
}
#[cfg(debug_assertions)]
block_on(typesetter.typeset(&src))
let start_layout = Instant::now();
let layouted = block_on(typesetter.layout(&parsed.output));
let layout = Instant::now() - start_layout;
let total = Instant::now() - start;
if !layouted.errors.is_empty() {
println!("layout errors: {:#?}", layouted.errors);
}
#[cfg(not(debug_assertions))] {
println!(" - cold start: {:?}", warmup);
println!(" - warmed up: {:?}", total);
println!(" - parsing: {:?}", parse);
println!(" - layouting: {:?}", layout);
println!();
}
layouted.output
}
/// Command line options.