konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Powerful parser testing 🐱‍👤

Browse Source
This commit is contained in:
Laurenz 2020-01-16 17:51:04 +01:00
parent 15ad30555b
commit 08b91a265f
23 changed files with 582 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.toml
View File

@ -19,17 +19,17 @@ default = ["fs-provider", "futures-executor"]
fs-provider = ["toddle/fs-provider"]
[[bin]]
name = "typst-bin"
name = "typst"
path = "src/bin/main.rs"
required-features = ["futures-executor"]
[[test]]
name = "layout"
path = "tests/layout.rs"
name = "layouter"
path = "tests/src/layouter.rs"
harness = false
required-features = ["futures-executor"]
[[test]]
name = "parse"
path = "tests/parse.rs"
name = "parser"
path = "tests/src/parser.rs"
harness = false

11
build.rs
View File

@ -1,19 +1,20 @@
use std::fs::{self, create_dir_all, read_dir, read_to_string};
use std::ffi::OsStr;
fn main() -> Result<(), Box<dyn std::error::Error>> {
create_dir_all("tests/cache")?;
// Make sure the script reruns if this file changes or files are
// added/deleted in the parsing folder.
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=tests/cache/parse");
println!("cargo:rerun-if-changed=tests/parsing");
println!("cargo:rerun-if-changed=tests/cache/parser-tests.rs");
println!("cargo:rerun-if-changed=tests/parser");
// Compile all parser tests into a single giant vector.
let mut code = "vec![".to_string();
for entry in read_dir("tests/parsing")? {
for entry in read_dir("tests/parser")? {
let path = entry?.path();
if path.extension() != Some(OsStr::new("rs")) {
continue;
@ -25,7 +26,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Make sure this also reruns if the contents of a file in parsing
// change. This is not ensured by rerunning only on the folder.
println!("cargo:rerun-if-changed=tests/parsing/{}.rs", name);
println!("cargo:rerun-if-changed=tests/parser/{}.rs", name);
code.push_str(&format!("(\"{}\", tokens!{{", name));
@ -44,7 +45,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
code.push(']');
fs::write("tests/cache/parse", code)?;
fs::write("tests/cache/parser-tests.rs", code)?;
Ok(())
}

41
src/func/macros.rs
View File

@ -52,46 +52,43 @@ macro_rules! function {
};
// (1-arg) Parse a parse-definition with only the first argument.
(@parse $type:ident $meta:ty | parse($args:ident) $code:block $($rest:tt)*) => {
function!(@parse $type $meta | parse($args, _body, _ctx, _meta) $code $($rest)*);
(@parse $type:ident $meta:ty | parse($header:ident) $code:block $($rest:tt)*) => {
function!(@parse $type $meta | parse($header, _body, _ctx, _meta) $code $($rest)*);
};
// (2-arg) Parse a parse-definition with only the first two arguments.
(@parse $type:ident $meta:ty |
parse($args:ident, $body:pat) $code:block $($rest:tt)*
parse($header:ident, $body:pat) $code:block $($rest:tt)*
) => {
function!(@parse $type $meta | parse($args, $body, _ctx, _meta) $code $($rest)*);
function!(@parse $type $meta | parse($header, $body, _ctx, _meta) $code $($rest)*);
};
// (3-arg) Parse a parse-definition with only the first three arguments.
(@parse $type:ident $meta:ty |
parse($args:ident, $body:pat, $ctx:pat) $code:block $($rest:tt)*
parse($header:ident, $body:pat, $ctx:pat) $code:block $($rest:tt)*
) => {
function!(@parse $type $meta | parse($args, $body, $ctx, _meta) $code $($rest)*);
function!(@parse $type $meta | parse($header, $body, $ctx, _meta) $code $($rest)*);
};
// (4-arg) Parse a parse-definition with all four arguments.
(@parse $type:ident $meta:ty |
parse($args:ident, $body:pat, $ctx:pat, $metadata:pat) $code:block
parse($header:ident, $body:pat, $ctx:pat, $metadata:pat) $code:block
$($rest:tt)*
) => {
use $crate::func::prelude::*;
impl $crate::func::ParseFunc for $type {
type Meta = $meta;
fn parse(
args: FuncArgs,
header: $crate::syntax::FuncHeader,
$body: Option<&str>,
$ctx: ParseContext,
$ctx: $crate::syntax::ParseContext,
$metadata: Self::Meta,
) -> ParseResult<Self> where Self: Sized {
) -> $crate::syntax::ParseResult<Self> where Self: Sized {
#[allow(unused_mut)]
let mut $args = args;
let mut $header = header;
let val = $code;
if !$args.is_empty() {
return Err($crate::TypesetError
::with_message("unexpected arguments"));
if !$header.args.is_empty() {
return Err($crate::TypesetError::with_message("unexpected arguments"));
}
Ok(val)
}
@ -112,14 +109,14 @@ macro_rules! function {
// (2-arg) Parse a layout-definition with all arguments.
(@layout $type:ident | layout($this:ident, $ctx:pat) $code:block) => {
use $crate::func::prelude::*;
impl LayoutFunc for $type {
impl $crate::func::LayoutFunc for $type {
fn layout<'a, 'life0, 'life1, 'async_trait>(
&'a $this,
$ctx: LayoutContext<'life0, 'life1>
) -> std::pin::Pin<Box<
dyn std::future::Future<Output = LayoutResult<Commands<'a>>> + 'async_trait
$ctx: $crate::layout::LayoutContext<'life0, 'life1>
) -> std::pin::Pin<Box<dyn std::future::Future<
Output = $crate::layout::LayoutResult<
$crate::func::Commands<'a>>
> + 'async_trait
>>
where
'a: 'async_trait,

11
src/func/mod.rs
View File

@ -14,12 +14,7 @@ mod macros;
pub mod prelude {
pub use crate::func::{Scope, ParseFunc, LayoutFunc, Command, Commands};
pub use crate::layout::prelude::*;
pub use crate::syntax::{
ParseContext, ParseResult,
SyntaxTree, FuncCall, FuncArgs,
Expression, Ident, ExpressionKind,
Spanned, Span
};
pub use crate::syntax::*;
pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize};
pub use crate::style::{LayoutStyle, PageStyle, TextStyle};
pub use Command::*;
@ -31,7 +26,7 @@ pub trait ParseFunc {
/// Parse the header and body into this function given a context.
fn parse(
args: FuncArgs,
header: FuncHeader,
body: Option<&str>,
ctx: ParseContext,
metadata: Self::Meta,
@ -125,7 +120,7 @@ pub struct Scope {
/// A function which parses the source of a function into a function type which
/// implements [`LayoutFunc`].
type Parser = dyn Fn(
FuncArgs,
FuncHeader,
Option<&str>,
ParseContext
) -> ParseResult<Box<dyn LayoutFunc>>;

4
src/library/align.rs
View File

@ -10,10 +10,10 @@ function! {
map: PosAxisMap<AlignmentKey>,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
AlignFunc {
body: parse!(optional: body, ctx),
map: PosAxisMap::new(&mut args)?,
map: PosAxisMap::new(&mut header.args)?,
}
}

6
src/library/boxed.rs
View File

@ -13,11 +13,11 @@ function! {
debug: Option<bool>,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
BoxFunc {
body: parse!(optional: body, ctx).unwrap_or(SyntaxTree::new()),
map: ExtentMap::new(&mut args, false)?,
debug: args.get_key_opt::<bool>("debug")?,
map: ExtentMap::new(&mut header.args, false)?,
debug: header.args.get_key_opt::<bool>("debug")?,
}
}

4
src/library/direction.rs
View File

@ -10,10 +10,10 @@ function! {
map: PosAxisMap<Direction>,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
DirectionFunc {
body: parse!(optional: body, ctx),
map: PosAxisMap::new(&mut args)?,
map: PosAxisMap::new(&mut header.args)?,
}
}

40
src/library/mod.rs
View File

@ -58,11 +58,11 @@ function! {
list: Vec<String>,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
FontFamilyFunc {
body: parse!(optional: body, ctx),
list: {
args.iter_pos().map(|arg| match arg.v {
header.args.iter_pos().map(|arg| match arg.v {
Expression::Str(s) |
Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
_ => error!("expected identifier or string"),
@ -86,11 +86,11 @@ function! {
style: FontStyle,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
FontStyleFunc {
body: parse!(optional: body, ctx),
style: {
let s = args.get_pos::<String>()?;
let s = header.args.get_pos::<String>()?;
match FontStyle::from_str(&s) {
Some(style) => style,
None => error!("invalid font style: `{}`", s),
@ -114,10 +114,10 @@ function! {
weight: FontWeight,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
FontWeightFunc {
body: parse!(optional: body, ctx),
weight: match args.get_pos::<Expression>()? {
weight: match header.args.get_pos::<Expression>()? {
Expression::Number(weight) => {
let weight = weight.round() as i16;
FontWeight(
@ -152,10 +152,10 @@ function! {
size: ScaleSize,
}
parse(args, body, ctx) {
parse(header, body, ctx) {
FontSizeFunc {
body: parse!(optional: body, ctx),
size: args.get_pos::<ScaleSize>()?,
size: header.args.get_pos::<ScaleSize>()?,
}
}
@ -187,11 +187,11 @@ function! {
type Meta = ContentKind;
parse(args, body, ctx, meta) {
parse(header, body, ctx, meta) {
ContentSpacingFunc {
body: parse!(optional: body, ctx),
content: meta,
spacing: args.get_pos::<f64>()? as f32,
spacing: header.args.get_pos::<f64>()? as f32,
}
}
@ -256,16 +256,18 @@ function! {
type Meta = Option<SpecificAxis>;
parse(args, body, _, meta) {
parse(header, body, _, meta) {
parse!(forbidden: body);
if let Some(axis) = meta {
SpacingFunc {
axis: AxisKey::Specific(axis),
spacing: FSize::from_expr(args.get_pos::<Spanned<Expression>>()?)?,
spacing: FSize::from_expr(
header.args.get_pos::<Spanned<Expression>>()?
)?,
}
} else {
for arg in args.iter_keys() {
for arg in header.args.iter_keys() {
let axis = AxisKey::from_ident(&arg.key)
.map_err(|_| error!(@unexpected_argument))?;
@ -295,16 +297,16 @@ function! {
Custom(ExtentMap<PSize>),
}
parse(args, body) {
parse(header, body) {
parse!(forbidden: body);
if let Some(name) = args.get_pos_opt::<Ident>()? {
let flip = args.get_key_opt::<bool>("flip")?.unwrap_or(false);
if let Some(name) = header.args.get_pos_opt::<Ident>()? {
let flip = header.args.get_key_opt::<bool>("flip")?.unwrap_or(false);
let paper = Paper::from_name(name.as_str())
.ok_or_else(|| error!(@"invalid paper name: `{}`", name))?;
PageSizeFunc::Paper(paper, flip)
} else {
PageSizeFunc::Custom(ExtentMap::new(&mut args, true)?)
PageSizeFunc::Custom(ExtentMap::new(&mut header.args, true)?)
}
}
@ -341,10 +343,10 @@ function! {
map: PaddingMap,
}
parse(args, body) {
parse(header, body) {
parse!(forbidden: body);
PageMarginsFunc {
map: PaddingMap::new(&mut args)?,
map: PaddingMap::new(&mut header.args)?,
}
}

3
src/syntax/color.rs
View File

@ -1,3 +0,0 @@
use super::*;

4
src/syntax/expr.rs
View File

@ -107,6 +107,10 @@ impl Object {
impl Display for Object {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
if self.pairs.len() == 0 {
return write!(f, "{{}}");
}
write!(f, "{{ ")?;
let mut first = true;

113
src/syntax/mod.rs
View File

@ -9,7 +9,6 @@ use crate::size::{Size, ScaleSize};
pub type ParseResult<T> = crate::TypesetResult<T>;
pub_use_mod!(color);
pub_use_mod!(expr);
pub_use_mod!(tokens);
pub_use_mod!(parsing);
@ -93,7 +92,7 @@ impl SyntaxTree {
}
/// A node in the syntax tree.
#[derive(Debug, PartialEq)]
#[derive(PartialEq)]
pub enum Node {
/// A number of whitespace characters containing less than two newlines.
Space,
@ -111,6 +110,28 @@ pub enum Node {
Func(FuncCall),
}
impl Display for Node {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Node::Space => write!(f, "Space"),
Node::Newline => write!(f, "Newline"),
Node::Text(text) => write!(f, "{:?}", text),
Node::ToggleItalic => write!(f, "ToggleItalic"),
Node::ToggleBolder => write!(f, "ToggleBold"),
Node::ToggleMonospace => write!(f, "ToggleMonospace"),
Node::Func(func) => {
if f.alternate() {
write!(f, "{:#?}", func.0)
} else {
write!(f, "{:?}", func.0)
}
}
}
}
}
debug_display!(Node);
/// An invocation of a function.
#[derive(Debug)]
pub struct FuncCall(pub Box<dyn LayoutFunc>);
@ -121,59 +142,20 @@ impl PartialEq for FuncCall {
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Colorization {
pub colors: Vec<Spanned<ColorToken>>,
}
/// Entities which can be colored by syntax highlighting.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum ColorToken {
Comment,
Bracket,
FuncName,
Colon,
Key,
Equals,
Comma,
Paren,
Brace,
ExprIdent,
ExprStr,
ExprNumber,
ExprSize,
ExprBool,
Bold,
Italic,
Monospace,
Invalid,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct ErrorMap {
pub errors: Vec<Spanned<String>>,
}
#[derive(Debug)]
#[derive(Debug, Clone, PartialEq)]
pub struct FuncHeader {
pub name: Spanned<Ident>,
pub args: FuncArgs,
}
#[derive(Debug)]
#[derive(Debug, Clone, PartialEq)]
pub struct FuncArgs {
positional: Tuple,
keyword: Object,
pub positional: Tuple,
pub keyword: Object,
}
impl FuncArgs {
fn new() -> FuncArgs {
pub fn new() -> FuncArgs {
FuncArgs {
positional: Tuple::new(),
keyword: Object::new(),
@ -258,3 +240,42 @@ fn expect<E: ExpressionKind>(opt: ParseResult<Option<E>>) -> ParseResult<E> {
Err(e) => Err(e),
}
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Colorization {
pub tokens: Vec<Spanned<ColorToken>>,
}
/// Entities which can be colored by syntax highlighting.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum ColorToken {
Comment,
Bracket,
FuncName,
Colon,
Key,
Equals,
Comma,
Paren,
Brace,
ExprIdent,
ExprStr,
ExprNumber,
ExprSize,
ExprBool,
Bold,
Italic,
Monospace,
Invalid,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct ErrorMap {
pub errors: Vec<Spanned<String>>,
}

27
src/syntax/parsing.rs
View File

@ -33,7 +33,7 @@ impl<'s> Parser<'s> {
src,
ctx,
error_map: ErrorMap { errors: vec![] },
colorization: Colorization { colors: vec![] },
colorization: Colorization { tokens: vec![] },
tokens: Tokens::new(src),
peeked: None,
@ -114,8 +114,6 @@ impl<'s> Parser<'s> {
}
fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> {
println!("peek: {:?}", self.peek());
let body = if self.peek() == Some(LeftBracket) {
self.eat();
@ -140,13 +138,15 @@ impl<'s> Parser<'s> {
};
let header = header?;
let name = header.name;
let parser = self.ctx.scope.get_parser(name.v.as_str()).or_else(|| {
self.error(format!("unknown function: `{}`", name.v), name.span);
let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| {
self.error(
format!("unknown function: `{}`", header.name.v),
header.name.span
);
None
})?;
Some(FuncCall(parser(header.args, body, self.ctx).unwrap()))
Some(FuncCall(parser(header, body, self.ctx).unwrap()))
}
fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
@ -163,16 +163,17 @@ impl<'s> Parser<'s> {
}
fn parse_func_args(&mut self) -> FuncArgs {
// unimplemented!()
// todo!()
self.eat_until(|t| t == RightBracket, true);
FuncArgs::new()
}
fn parse_tuple(&mut self) -> Spanned<Expression> {
unimplemented!("parse_tuple")
todo!("parse_tuple")
}
fn parse_object(&mut self) -> Spanned<Expression> {
unimplemented!("parse_object")
todo!("parse_object")
}
fn skip_whitespace(&mut self) {
@ -207,13 +208,13 @@ impl<'s> Parser<'s> {
fn color(&mut self, token: Spanned<ColorToken>, replace_last: bool) {
if replace_last {
if let Some(last) = self.colorization.colors.last_mut() {
if let Some(last) = self.colorization.tokens.last_mut() {
*last = token;
return;
}
}
self.colorization.colors.push(token);
self.colorization.tokens.push(token);
}
fn color_token(&mut self, token: Spanned<Token<'s>>) {
@ -235,7 +236,7 @@ impl<'s> Parser<'s> {
};
if let Some(color) = colored {
self.colorization.colors.push(Spanned { v: color, span: token.span });
self.colorization.tokens.push(Spanned { v: color, span: token.span });
}
}

18
src/syntax/span.rs
View File

@ -1,6 +1,6 @@
//! Spans map elements to the part of source code they originate from.
use std::fmt::{self, Display, Formatter};
use std::fmt::{self, Debug, Display, Formatter};
/// Annotates a value with the part of the source code it corresponds to.
@ -28,13 +28,21 @@ impl<T> Spanned<T> {
}
}
impl<T> Display for Spanned<T> where T: std::fmt::Debug {
impl<T> Display for Spanned<T> where T: std::fmt::Display {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "({:?}:{})", self.v, self.span)
write!(f, "({}, {}, ", self.span.start, self.span.end)?;
self.v.fmt(f)?;
write!(f, ")")
}
}
debug_display!(Spanned; T where T: std::fmt::Debug);
impl<T> Debug for Spanned<T> where T: std::fmt::Debug {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "({}, {}, ", self.span.start, self.span.end)?;
self.v.fmt(f)?;
write!(f, ")")
}
}
/// Describes a slice of source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
@ -68,7 +76,7 @@ impl Span {
impl Display for Span {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "[{}, {}]", self.start, self.end)
write!(f, "({}, {})", self.start, self.end)
}
}

0
tests/layouts/coma.typ → tests/layouter/coma.typ
View File

0
tests/layouts/stack.typ → tests/layouter/stack.typ
View File

236
tests/parse.rs
View File

@ -1,236 +0,0 @@
#![allow(unused_imports)]
#![allow(dead_code)]
#![allow(non_snake_case)]
use typstc::func::Scope;
use typstc::size::Size;
use typstc::syntax::*;
use typstc::{function, parse};
mod token_shorthands {
pub use super::Token::{
Whitespace as W,
LineComment as LC, BlockComment as BC, StarSlash as SS,
LeftBracket as LB, RightBracket as RB,
LeftParen as LP, RightParen as RP,
LeftBrace as LBR, RightBrace as RBR,
Colon as CL, Comma as CM, Equals as EQ,
ExprIdent as ID, ExprStr as STR, ExprSize as SIZE,
ExprNumber as NUM, ExprBool as BOOL,
Star as ST, Underscore as U, Backtick as B, Text as T,
};
}
mod node_shorthands {
use super::Node;
pub use Node::{
Space as S, Newline as N, Text,
ToggleItalic as I, ToggleBolder as B, ToggleMonospace as M,
Func,
};
pub fn T(text: &str) -> Node { Node::Text(text.to_string()) }
}
macro_rules! F {
(@body None) => (None);
(@body Some([$($tts:tt)*])) => ({
let nodes = vec![$($tts)*].into_iter()
.map(|v| Spanned { v, span: Span::ZERO })
.collect();
Some(SyntaxTree { nodes })
});
($($body:tt)*) => ({
Func(FuncCall(Box::new(DebugFn {
pos: vec![],
key: vec![],
body: F!(@body $($body)*),
})))
});
}
function! {
#[derive(Debug, PartialEq)]
pub struct DebugFn {
pos: Vec<Spanned<Expression>>,
key: Vec<Pair>,
body: Option<SyntaxTree>,
}
parse(args, body, ctx) {
DebugFn {
pos: args.iter_pos().collect(),
key: args.iter_keys().collect(),
body: parse!(optional: body, ctx),
}
}
layout() { vec![] }
}
impl DebugFn {
fn compare(&self, other: &DebugFn) -> bool {
self.pos.iter().zip(&other.pos).all(|(a, b)| a.v == b.v)
&& self.key.iter().zip(&other.key)
.all(|(a, b)| a.key.v == b.key.v && a.value.v == b.value.v)
&& match (&self.body, &other.body) {
(Some(a), Some(b)) => compare(a, b),
(None, None) => true,
_ => false,
}
}
}
fn downcast(func: &FuncCall) -> &DebugFn {
func.0.downcast::<DebugFn>().expect("not a debug fn")
}
fn compare(a: &SyntaxTree, b: &SyntaxTree) -> bool {
for (x, y) in a.nodes.iter().zip(&b.nodes) {
use node_shorthands::*;
let same = match (&x.v, &y.v) {
(S, S) | (N, N) | (I, I) | (B, B) | (M, M) => true,
(Text(t1), Text(t2)) => t1 == t2,
(Func(f1), Func(f2)) => {
downcast(f1).compare(downcast(f2))
}
_ => false,
};
if !same { return false; }
}
true
}
/// Parses the test syntax.
macro_rules! tokens {
($($task:ident $src:expr =>($line:expr)=> [$($tts:tt)*])*) => ({
#[allow(unused_mut)]
let mut cases = Vec::new();
$(cases.push(($line, $src, tokens!(@$task [$($tts)*])));)*
cases
});
(@t [$($tts:tt)*]) => ({
use token_shorthands::*;
Target::Tokenize(vec![$($tts)*])
});
(@ts [$($tts:tt)*]) => ({
use token_shorthands::*;
Target::TokenizeSpanned(tokens!(@__spans [$($tts)*]))
});
(@p [$($tts:tt)*]) => ({
use node_shorthands::*;
let nodes = vec![$($tts)*].into_iter()
.map(|v| Spanned { v, span: Span::ZERO })
.collect();
Target::Parse(SyntaxTree { nodes })
});
(@ps [$($tts:tt)*]) => ({
use node_shorthands::*;
Target::ParseSpanned(tokens!(@__spans [$($tts)*]))
});
(@__spans [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?]) => ({
vec![
$(Spanned { v: $v, span: Span {
start: Position { line: $sl, column: $sc },
end: Position { line: $el, column: $ec },
}}),*
]
});
}
#[derive(Debug)]
enum Target {
Tokenize(Vec<Token<'static>>),
TokenizeSpanned(Vec<Spanned<Token<'static>>>),
Parse(SyntaxTree),
ParseSpanned(SyntaxTree),
}
fn main() {
let tests = include!("cache/parse");
let mut errors = false;
let len = tests.len();
println!();
println!("Running {} test{}", len, if len > 1 { "s" } else { "" });
// Go through all test files.
for (file, cases) in tests.into_iter() {
print!("Testing: {}. ", file);
let mut okay = 0;
let mut failed = 0;
// Go through all tests in a test file.
for (line, src, target) in cases.into_iter() {
let (correct, expected, found) = test_case(src, target);
// Check whether the tokenization works correctly.
if correct {
okay += 1;
} else {
if failed == 0 {
println!();
}
println!(" - Case failed in file {}.rs in line {}.", file, line);
println!(" - Source: {:?}", src);
println!(" - Expected: {:?}", expected);
println!(" - Found: {:?}", found);
println!();
failed += 1;
errors = true;
}
}
// Print a small summary.
print!("{} okay, {} failed.", okay, failed);
if failed == 0 {
print!(" ✔")
}
println!();
}
println!();
if errors {
std::process::exit(-1);
}
}
fn test_case(src: &str, target: Target) -> (bool, String, String) {
match target {
Target::Tokenize(tokens) => {
let found: Vec<_> = tokenize(src).map(Spanned::value).collect();
(found == tokens, format!("{:?}", tokens), format!("{:?}", found))
}
Target::TokenizeSpanned(tokens) => {
let found: Vec<_> = tokenize(src).collect();
(found == tokens, format!("{:?}", tokens), format!("{:?}", found))
}
Target::Parse(tree) => {
let scope = Scope::with_debug::<DebugFn>();
let (found, _, errs) = parse(src, ParseContext { scope: &scope });
(compare(&tree, &found), format!("{:?}", tree), format!("{:?}", found))
}
Target::ParseSpanned(tree) => {
let scope = Scope::with_debug::<DebugFn>();
let (found, _, _) = parse(src, ParseContext { scope: &scope });
(tree == found, format!("{:?}", tree), format!("{:?}", found))
}
}
}

8
tests/parsing/tokens.rs → tests/parser/tokens.rs
View File

@ -41,13 +41,13 @@ t "[a: true, x=1]" => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
t "[120%]" => [LB, NUM(1.2), RB]
// Body only tokens.
t "_*`" => [U, ST, B]
t "[func]*bold*" => [LB, ID("func"), RB, ST, T("bold"), ST]
t "_*`" => [U, S, B]
t "[func]*bold*" => [LB, ID("func"), RB, S, T("bold"), S]
t "[_*`]" => [LB, T("_"), T("*"), T("`"), RB]
t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
// Nested functions.
t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, ST, RB, RB]
t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB]
t "[_][[,],]," => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")]
t "[=][=][=]" => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB]
t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB]
@ -75,6 +75,6 @@ ts "[a=10]" => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ),
(0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)]
ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)),
(0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")),
(0:10, 0:11, RB), (0:11, 0:12, ST)]
(0:10, 0:11, RB), (0:11, 0:12, S)]
ts "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))]
ts "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)]

33
tests/parser/trees.rs Normal file
View File

@ -0,0 +1,33 @@
p "" => []
p "hi" => [T("hi")]
p "hi you" => [T("hi"), S, T("you")]
p "❤\n\n 🌍" => [T("❤"), N, T("🌍")]
p "[func]" => [func!("func"; None)]
p "[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, B, T("you"), B]))]
p "from [align: left] to" => [
T("from"), S, func!("align", pos: [ID("left")]; None), S, T("to"),
]
p "[box: x=1.2pt, false][a b c] bye" => [
func!(
"box",
pos: [BOOL(false)],
key: ["x" => SIZE(Size::pt(1.2))];
Some([T("a"), S, T("b"), S, T("c")])
),
S, T("bye"),
]
c "hi" => []
c "[align: left][\n _body_\n]" => [
(0:0, 0:1, B),
(0:1, 0:6, FN),
(0:6, 0:7, CL),
(0:8, 0:12, ID),
(0:12, 0:13, B),
(0:13, 0:14, B),
(1:4, 1:10, IT),
(2:0, 2:2, B),
]

20
tests/parsing/trees.rs
View File

@ -1,20 +0,0 @@
p "" => []
p "hi" => [T("hi")]
p "hi you" => [T("hi"), S, T("you")]
p "❤\n\n 🌍" => [T("❤"), N, T("🌍")]
p "[func]" => [F!(None)]
p "[tree][hi *you*]" => [F!(Some([T("hi"), S, B, T("you"), B]))]
// p "from [align: left] to" => [
// T("from"), S,
// F!("align", pos=[ID("left")], None),
// S, T("to"),
// ]
// p "[box: x=1.2pt, false][a b c] bye" => [
// F!(
// "box",
// pos=[BOOL(false)],
// key=["x": SIZE(Size::pt(1.2))],
// Some([T("a"), S, T("b"), S, T("c")]),
// ),
// S, T("bye"),
// ]

9
tests/layout.rs → tests/src/layouter.rs
View File

@ -15,16 +15,17 @@ use typstc::style::PageStyle;
use typstc::toddle::query::FileSystemFontProvider;
use typstc::export::pdf::PdfExporter;
type Result<T> = std::result::Result<T, Box<dyn Error>>;
fn main() -> Result<()> {
type DynResult<T> = Result<T, Box<dyn Error>>;
fn main() -> DynResult<()> {
let opts = Options::parse();
create_dir_all("tests/cache/serial")?;
create_dir_all("tests/cache/render")?;
create_dir_all("tests/cache/pdf")?;
let tests: Vec<_> = read_dir("tests/layouts/")?.collect();
let tests: Vec<_> = read_dir("tests/layouter/")?.collect();
let mut filtered = Vec::new();
for entry in tests {
@ -62,7 +63,7 @@ fn main() -> Result<()> {
}
/// Create a _PDF_ with a name from the source code.
fn test(name: &str, src: &str) -> Result<()> {
fn test(name: &str, src: &str) -> DynResult<()> {
println!("Testing: {}.", name);
let mut typesetter = Typesetter::new();

311
tests/src/parser.rs Normal file
View File

@ -0,0 +1,311 @@
use std::fmt::Debug;
use typstc::func::Scope;
use typstc::size::Size;
use typstc::syntax::*;
use typstc::{function, parse};
mod spanless;
use spanless::SpanlessEq;
/// The result of a single test case.
enum Case {
Okay,
Failed {
line: usize,
src: &'static str,
expected: String,
found: String,
}
}
/// Test all tests.
fn test(tests: Vec<(&str, Vec<Case>)>) {
println!();
let mut errors = false;
let len = tests.len();
println!("Running {} test{}", len, if len > 1 { "s" } else { "" });
for (file, cases) in tests {
print!("Testing: {}. ", file);
let mut okay = 0;
let mut failed = 0;
for case in cases {
match case {
Case::Okay => okay += 1,
Case::Failed { line, src, expected, found } => {
println!();
println!(" - Case failed in file {}.rs in line {}.", file, line);
println!(" - Source: {:?}", src);
println!(" - Expected: {}", expected);
println!(" - Found: {}", found);
failed += 1;
}
}
}
// Print a small summary.
print!("{} okay, {} failed.", okay, failed);
if failed == 0 {
print!(" ✔")
} else {
errors = true;
}
println!();
}
println!();
if errors {
std::process::exit(-1);
}
}
/// The main test macro.
macro_rules! tokens {
($($task:ident $src:expr =>($line:expr)=> [$($e:tt)*])*) => ({
vec![$({
let (okay, expected, found) = case!($task $src, [$($e)*]);
if okay {
Case::Okay
} else {
Case::Failed {
line: $line,
src: $src,
expected: format(expected),
found: format(found),
}
}
}),*]
});
}
//// Indented formatting for failed cases.
fn format(thing: impl Debug) -> String {
format!("{:#?}", thing).replace('\n', "\n ")
}
/// Evaluates a single test.
macro_rules! case {
(t $($rest:tt)*) => (case!(@tokenize SpanlessEq::spanless_eq, $($rest)*));
(ts $($rest:tt)*) => (case!(@tokenize PartialEq::eq, $($rest)*));
(@tokenize $cmp:expr, $src:expr, [$($e:tt)*]) => ({
let expected = list!(tokens [$($e)*]);
let found = tokenize($src).collect::<Vec<_>>();
($cmp(&found, &expected), expected, found)
});
(p $($rest:tt)*) => (case!(@parse SpanlessEq::spanless_eq, $($rest)*));
(ps $($rest:tt)*) => (case!(@parse PartialEq::eq, $($rest)*));
(@parse $cmp:expr, $src:expr, [$($e:tt)*]) => ({
let expected = SyntaxTree { nodes: list!(nodes [$($e)*]) };
let found = parse($src, ParseContext { scope: &scope() }).0;
($cmp(&found, &expected), expected, found)
});
(c $src:expr, [$($e:tt)*]) => ({
let expected = Colorization { tokens: list!(colors [$($e)*]) };
let found = parse($src, ParseContext { scope: &scope() }).1;
(expected == found, expected, found)
});
(e $src:expr, [$($e:tt)*]) => ({
let expected = ErrorMap { errors: list!([$($e)*]) };
let found = parse($src, ParseContext { scope: &scope() }).2;
(expected == found, expected, found)
});
}
/// A scope containing the `DebugFn` as a fallback.
fn scope() -> Scope {
Scope::with_debug::<DebugFn>()
}
/// Parses possibly-spanned lists of token or node expressions.
macro_rules! list {
(expr [$($item:expr),* $(,)?]) => ({
#[allow(unused_imports)]
use cuts::expr::*;
Tuple { items: vec![$(zspan($item)),*] }
});
(expr [$($key:expr =>($_:expr)=> $value:expr),* $(,)?]) => ({
#[allow(unused_imports)]
use cuts::expr::*;
Object {
pairs: vec![$(Pair {
key: zspan(Ident($key.to_string())),
value: zspan($value),
}),*]
}
});
($cut:ident [$($e:tt)*]) => ({
#[allow(unused_imports)]
use cuts::$cut::*;
list!([$($e)*])
});
([$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?]) => ({
vec![
$(Spanned { v: $v, span: Span {
start: Position { line: $sl, column: $sc },
end: Position { line: $el, column: $ec },
}}),*
]
});
([$($e:tt)*]) => (vec![$($e)*].into_iter().map(zspan).collect::<Vec<_>>());
}
/// Composes a function expression.
macro_rules! func {
($name:expr $(,pos: [$($p:tt)*])? $(,key: [$($k:tt)*])?; $($b:tt)*) => ({
#![allow(unused_mut, unused_assignments)]
let mut positional = Tuple::new();
let mut keyword = Object::new();
$(positional = list!(expr [$($p)*]);)?
$(keyword = list!(expr [$($k)*]);)?
Node::Func(FuncCall(Box::new(DebugFn {
header: FuncHeader {
name: zspan(Ident($name.to_string())),
args: FuncArgs {
positional,
keyword,
},
},
body: func!(@body $($b)*),
})))
});
(@body Some($($b:tt)*)) => (Some(SyntaxTree { nodes: list!(nodes $($b)*) }));
(@body None) => (None);
}
function! {
/// Most functions in the tests are parsed into the debug function for easy
/// inspection of arguments and body.
#[derive(Debug, PartialEq)]
pub struct DebugFn {
header: FuncHeader,
body: Option<SyntaxTree>,
}
parse(header, body, ctx) {
DebugFn {
header: header.clone(),
body: parse!(optional: body, ctx),
}
}
layout() { vec![] }
}
/// Span an element with a zero span.
fn zspan<T>(v: T) -> Spanned<T> {
Spanned { v, span: Span::ZERO }
}
/// Abbreviations for tokens, nodes, colors and expressions.
#[allow(non_snake_case, dead_code)]
mod cuts {
pub mod tokens {
pub use typstc::syntax::Token::{
Whitespace as W,
LineComment as LC,
BlockComment as BC,
StarSlash as SS,
LeftBracket as LB,
RightBracket as RB,
LeftParen as LP,
RightParen as RP,
LeftBrace as LBR,
RightBrace as RBR,
Colon as CL,
Comma as CM,
Equals as EQ,
ExprIdent as ID,
ExprStr as STR,
ExprSize as SIZE,
ExprNumber as NUM,
ExprBool as BOOL,
Star as S,
Underscore as U,
Backtick as B,
Text as T,
};
}
pub mod nodes {
use typstc::syntax::Node;
pub use Node::{
Space as S,
Newline as N,
ToggleItalic as I,
ToggleBolder as B,
ToggleMonospace as M,
};
pub fn T(text: &str) -> Node {
Node::Text(text.to_string())
}
}
pub mod colors {
pub use typstc::syntax::ColorToken::{
Comment as C,
Bracket as B,
FuncName as FN,
Colon as CL,
Key as K,
Equals as EQ,
Comma as CM,
Paren as P,
Brace as BR,
ExprIdent as ID,
ExprStr as STR,
ExprNumber as NUM,
ExprSize as SIZE,
ExprBool as BOOL,
Bold as BD,
Italic as IT,
Monospace as MS,
Invalid as INV,
};
}
pub mod expr {
use typstc::syntax::{Expression, Ident};
pub use Expression::{
Number as NUM,
Size as SIZE,
Bool as BOOL,
};
pub fn ID(text: &str) -> Expression {
Expression::Ident(Ident(text.to_string()))
}
pub fn STR(text: &str) -> Expression {
Expression::Str(text.to_string())
}
}
}
fn main() {
test(include!("../cache/parser-tests.rs"))
}

14
tests/render.py → tests/src/render.py
View File

@ -7,7 +7,7 @@ from PIL import Image, ImageDraw, ImageFont
BASE = os.path.dirname(__file__)
CACHE = os.path.join(BASE, 'cache/')
CACHE = os.path.join(BASE, '../cache/')
SERIAL = os.path.join(CACHE, 'serial/')
RENDER = os.path.join(CACHE, 'render/')
@ -98,16 +98,18 @@ class MultiboxRenderer:
class BoxRenderer:
def __init__(self, fonts, width, height):
def __init__(self, fonts, width, height, grid=False):
self.fonts = fonts
self.size = (pix(width), pix(height))
img = Image.new('RGBA', self.size, (255, 255, 255, 255))
pixels = numpy.array(img)
# for i in range(0, int(height)):
# for j in range(0, int(width)):
# if ((i // 2) % 2 == 0) == ((j // 2) % 2 == 0):
# pixels[4*i:4*(i+1), 4*j:4*(j+1)] = (225, 225, 225, 255)
if grid:
for i in range(0, int(height)):
for j in range(0, int(width)):
if ((i // 2) % 2 == 0) == ((j // 2) % 2 == 0):
pixels[4*i:4*(i+1), 4*j:4*(j+1)] = (225, 225, 225, 255)
self.img = Image.fromarray(pixels, 'RGBA')
self.draw = ImageDraw.Draw(self.img)

62
tests/src/spanless.rs Normal file
View File

@ -0,0 +1,62 @@
use super::*;
/// Compares elements by only looking at values and ignoring spans.
pub trait SpanlessEq<T> {
fn spanless_eq(&self, other: &T) -> bool;
}
impl SpanlessEq<Vec<Spanned<Token<'_>>>> for Vec<Spanned<Token<'_>>> {
fn spanless_eq(&self, other: &Vec<Spanned<Token>>) -> bool {
self.len() == other.len()
&& self.iter().zip(other).all(|(x, y)| x.v == y.v)
}
}
impl SpanlessEq<SyntaxTree> for SyntaxTree {
fn spanless_eq(&self, other: &SyntaxTree) -> bool {
fn downcast(func: &FuncCall) -> &DebugFn {
func.0.downcast::<DebugFn>().expect("not a debug fn")
}
self.nodes.len() == other.nodes.len()
&& self.nodes.iter().zip(&other.nodes).all(|(x, y)| match (&x.v, &y.v) {
(Node::Func(a), Node::Func(b)) => downcast(a).spanless_eq(downcast(b)),
(a, b) => a == b,
})
}
}
impl SpanlessEq<DebugFn> for DebugFn {
fn spanless_eq(&self, other: &DebugFn) -> bool {
self.header.name.v == other.header.name.v
&& self.header.args.positional.spanless_eq(&other.header.args.positional)
&& self.header.args.keyword.spanless_eq(&other.header.args.keyword)
}
}
impl SpanlessEq<Expression> for Expression {
fn spanless_eq(&self, other: &Expression) -> bool {
match (self, other) {
(Expression::Tuple(a), Expression::Tuple(b)) => a.spanless_eq(b),
(Expression::Object(a), Expression::Object(b)) => a.spanless_eq(b),
(a, b) => a == b,
}
}
}
impl SpanlessEq<Tuple> for Tuple {
fn spanless_eq(&self, other: &Tuple) -> bool {
self.items.len() == other.items.len()
&& self.items.iter().zip(&other.items)
.all(|(x, y)| x.v.spanless_eq(&y.v))
}
}
impl SpanlessEq<Object> for Object {
fn spanless_eq(&self, other: &Object) -> bool {
self.pairs.len() == other.pairs.len()
&& self.pairs.iter().zip(&other.pairs)
.all(|(x, y)| x.key.v == y.key.v && x.value.v.spanless_eq(&y.value.v))
}
}