Create parsing test harness ⚡

Cargo.toml

@@ -3,6 +3,7 @@ name = "typstc"
 version = "0.1.0"
 authors = ["Laurenz Mädje <laurmaedje@gmail.com>"]
 edition = "2018"
+build = "build.rs"
 
 [dependencies]
 tide = { path = "../tide" }
@@ -16,6 +17,11 @@ name = "typst-bin"
 path = "src/bin/main.rs"
 
 [[test]]
-name = "layouting"
-path = "tests/layouting.rs"
+name = "layout"
+path = "tests/layout.rs"
+harness = false
+
+[[test]]
+name = "parse"
+path = "tests/parse.rs"
 harness = false

build.rs

@@ -0,0 +1,34 @@
+use std::fs;
+use std::ffi::OsStr;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=tests/parsing");
+
+    fs::create_dir_all("tests/cache").unwrap();
+
+    let paths = fs::read_dir("tests/parsing").unwrap()
+        .map(|entry| entry.unwrap().path())
+        .filter(|path| path.extension() == Some(OsStr::new("rs")));
+
+    let mut code = "vec![".to_string();
+    for path in paths {
+        let name = path.file_stem().unwrap().to_str().unwrap();
+        let file = fs::read_to_string(&path).unwrap();
+
+        println!("cargo:rerun-if-changed=tests/parsing/{}.rs", name);
+
+        code.push_str(&format!("(\"{}\", tokens!{{", name));
+
+        for (index, line) in file.lines().enumerate() {
+            let mut line = line.replace("=>", &format!("=>({})=>", index + 1));
+            line.push('\n');
+            code.push_str(&line);
+        }
+
+        code.push_str("}),");
+    }
+    code.push(']');
+
+    fs::write("tests/cache/parsing.rs", code).unwrap();
+}

src/syntax/tokens.rs

@@ -204,7 +204,7 @@ impl<'s> Iterator for Tokens<'s> {
             '\\' => {
                 if let Some((index, c)) = self.chars.peek() {
                     let escapable = match c {
-                        '[' | ']' | '\\' | '*' | '_' | '`' | ':' | '=' | '/' => true,
+                        '[' | ']' | '\\' | '*' | '_' | '`' | ':' | '=' | ',' | '/' => true,
                         _ => false,
                     };
 

tests/layout.rs

@@ -31,7 +31,7 @@ fn main() {
 
     let mut failed = 0;
 
-    for entry in fs::read_dir("tests/layouts/").unwrap() {
+    for entry in fs::read_dir("tests/layouting/").unwrap() {
         let path = entry.unwrap().path();
 
         if path.extension() != Some(std::ffi::OsStr::new("typ")) {
@@ -60,8 +60,11 @@ fn main() {
 
     if failed > 0 {
         println!("{} tests failed.", failed);
+        println!();
         std::process::exit(-1);
     }
+
+    println!();
 }
 
 /// Create a _PDF_ with a name from the source code.

tests/parse.rs

@@ -0,0 +1,59 @@
+use typstc::syntax::*;
+
+use Token::{
+    Space as S, Newline as N, LeftBracket as LB,
+    RightBracket as RB, Text as T, *
+};
+
+macro_rules! tokens {
+    ($($src:expr =>($line:expr)=> $tokens:expr)*) => ({
+        #[allow(unused_mut)]
+        let mut cases = Vec::new();
+        $(cases.push(($line, $src, $tokens.to_vec()));)*
+        cases
+    });
+}
+
+fn main() {
+    let tests = include!("cache/parsing.rs");
+
+    let mut errors = false;
+    for (file, cases) in tests.into_iter() {
+        print!("Testing: {}. ", file);
+
+        let mut okay = 0;
+        let mut failed = 0;
+
+        for (line, src, expected) in cases.into_iter() {
+            let found: Vec<_> = tokenize(src).map(Spanned::value).collect();
+
+            if found == expected {
+                okay += 1;
+            } else {
+                if failed == 0 {
+                    println!();
+                }
+
+                println!(" - Case failed in file {}.rs in line {}.", file, line);
+                println!("   - Source:   {:?}", src);
+                println!("   - Expected: {:?}", expected);
+                println!("   - Found:    {:?}", found);
+
+                failed += 1;
+                errors = true;
+            }
+        }
+
+        print!("{} okay, {} failed.", okay, failed);
+        if failed == 0 {
+            print!(" ✔")
+        }
+        println!();
+    }
+
+    println!();
+
+    if errors {
+        std::process::exit(-1);
+    }
+}

tests/parsing/base.rs

@@ -0,0 +1,78 @@
+// Spaces, Newlines, Brackets.
+""                => []
+" "               => [S]
+"    "            => [S]
+"\t"              => [S]
+"  \t"            => [S]
+"\n"              => [N]
+"\n "             => [N, S]
+"  \n"            => [S, N]
+"  \n   "         => [S, N, S]
+"["               => [LB]
+"]"               => [RB]
+
+// Header only tokens.
+"[:]"             => [LB, Colon, RB]
+"[=]"             => [LB, Equals, RB]
+"[,]"             => [LB, Comma, RB]
+":"               => [T(":")]
+"="               => [T("=")]
+","               => [T(",")]
+r#"["hi"]"#       => [LB, Quoted("hi"), RB]
+r#""hi""#         => [T(r#""hi""#)]
+
+// Body only tokens.
+"_"               => [Underscore]
+"*"               => [Star]
+"`"               => [Backtick]
+"[_]"             => [LB, T("_"), RB]
+"[*]"             => [LB, T("*"), RB]
+"[`]"             => [LB, T("`"), RB]
+
+// Comments.
+"//line"          => [LineComment("line")]
+"/*block*/"       => [BlockComment("block")]
+"*/"              => [StarSlash]
+
+// Plain text.
+"A"               => [T("A")]
+"Hello"           => [T("Hello")]
+"Hello-World"     => [T("Hello-World")]
+r#"A"B"#          => [T(r#"A"B"#)]
+"🌍"              => [T("🌍")]
+
+// Escapes.
+r"\["             => [T("[")]
+r"\]"             => [T("]")]
+r"\\"             => [T(r"\")]
+r"[\[]"           => [LB, T("["), RB]
+r"[\]]"           => [LB, T("]"), RB]
+r"[\\]"           => [LB, T(r"\"), RB]
+r"\:"             => [T(":")]
+r"\="             => [T("=")]
+r"\/"             => [T("/")]
+r"[\:]"           => [LB, T(":"), RB]
+r"[\=]"           => [LB, T("="), RB]
+r"[\,]"           => [LB, T(","), RB]
+r"\*"             => [T("*")]
+r"\_"             => [T("_")]
+r"\`"             => [T("`")]
+r"[\*]"           => [LB, T("*"), RB]
+r"[\_]"           => [LB, T("_"), RB]
+r"[\`]"           => [LB, T("`"), RB]
+
+// Whitespace.
+"Hello World"     => [T("Hello"), S, T("World")]
+"Hello  World"    => [T("Hello"), S, T("World")]
+"Hello \t World"  => [T("Hello"), S, T("World")]
+
+// Newline.
+"First\n"         => [T("First"), N]
+"First \n"        => [T("First"), S, N]
+"First\n "        => [T("First"), N, S]
+"First \n "       => [T("First"), S, N, S]
+"First\nSecond"   => [T("First"), N, T("Second")]
+"First\r\nSecond" => [T("First"), N, T("Second")]
+"First \nSecond"  => [T("First"), S, N, T("Second")]
+"First\n Second"  => [T("First"), N, S, T("Second")]
+"First \n Second" => [T("First"), S, N, S, T("Second")]