konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge some modules

Browse Source
This commit is contained in:
Laurenz 2022-10-17 19:26:24 +02:00
parent 4fd031a256
commit e218226655
45 changed files with 802 additions and 823 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Cargo.lock generated
View File

@ -374,15 +374,6 @@ dependencies = [
"libc",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "getrandom"
version = "0.2.7"
@ -1126,7 +1117,6 @@ dependencies = [
"dirs",
"elsa",
"flate2",
"fxhash",
"hypher",
"iai",
"image",

63
Cargo.toml
View File

@ -11,25 +11,23 @@ typst-macros = { path = "./macros" }
# Utilities
bitflags = "1"
bytemuck = "1"
fxhash = "0.2"
comemo = "0.1"
once_cell = "1"
regex = "1"
serde = { version = "1", features = ["derive"] }
siphasher = "0.3"
typed-arena = "2"
unscanny = "0.1"
regex = "1"
# Incremental compilation
comemo = "0.1"
# Text and font handling
hypher = "0.1"
kurbo = "0.8"
ttf-parser = "0.17"
rustybuzz = "0.5"
ttf-parser = "0.17"
unicode-bidi = "0.3.5"
unicode-script = "0.5"
unicode-segmentation = "1"
unicode-xid = "0.2"
unicode-script = "0.5"
xi-unicode = "0.3"
# Raster and vector graphics handling
@ -37,12 +35,12 @@ image = { version = "0.24", default-features = false, features = ["png", "jpeg",
usvg = { version = "0.22", default-features = false }
# External implementation of user-facing features
syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] }
rex = { git = "https://github.com/laurmaedje/ReX" }
unicode-math = { git = "https://github.com/s3bk/unicode-math/" }
lipsum = { git = "https://github.com/reknih/lipsum" }
csv = "1"
lipsum = { git = "https://github.com/reknih/lipsum" }
rex = { git = "https://github.com/laurmaedje/ReX" }
serde_json = "1"
syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] }
unicode-math = { git = "https://github.com/s3bk/unicode-math/" }
# PDF export
miniz_oxide = "0.5"
@ -50,52 +48,50 @@ pdf-writer = "0.6"
subsetter = "0.1"
svg2pdf = "0.4"
# Raster export / rendering
tiny-skia = "0.6.2"
# Rendering
flate2 = "1"
pixglyph = { git = "https://github.com/typst/pixglyph" }
resvg = { version = "0.22", default-features = false }
roxmltree = "0.14"
flate2 = "1"
tiny-skia = "0.6.2"
# Command line interface / tests
pico-args = { version = "0.4", optional = true }
# Command line interface
chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true }
codespan-reporting = { version = "0.11", optional = true }
dirs = { version = "4", optional = true }
elsa = { version = "1.7", optional = true }
memmap2 = { version = "0.5", optional = true }
notify = { version = "5", optional = true }
pico-args = { version = "0.4", optional = true }
same-file = { version = "1", optional = true }
walkdir = { version = "2", optional = true }
elsa = { version = "1.7", optional = true }
dirs = { version = "4", optional = true }
memmap2 = { version = "0.5", optional = true }
siphasher = { version = "0.3", optional = true }
notify = { version = "5", optional = true }
chrono = { version = "0.4", default-features = false, features = ["clock", "std"], optional = true }
[dev-dependencies]
iai = { git = "https://github.com/reknih/iai" }
walkdir = "2"
elsa = "1.7"
[workspace]
members = ["macros"]
[features]
default = ["tests"]
tests = ["same-file", "walkdir", "elsa", "siphasher"]
cli = [
"pico-args",
"chrono",
"codespan-reporting",
"dirs",
"elsa",
"memmap2",
"notify",
"pico-args",
"same-file",
"walkdir",
"elsa",
"siphasher",
"notify",
"chrono",
]
[profile.dev]
# Faster compilation
debug = 0
debug = 0 # Faster compilation
[profile.dev.package."*"]
# Faster test execution
opt-level = 2
opt-level = 2 # Faster test execution
[[bin]]
name = "typst"
@ -103,7 +99,6 @@ required-features = ["cli"]
[[test]]
name = "typeset"
required-features = ["tests"]
harness = false
[[bench]]

13
benches/oneshot.rs
View File

@ -6,8 +6,7 @@ use unscanny::Scanner;
use typst::diag::{FileError, FileResult};
use typst::font::{Font, FontBook};
use typst::parse::{TokenMode, Tokens};
use typst::source::{Source, SourceId};
use typst::syntax::{Source, SourceId, TokenMode, Tokens};
use typst::util::Buffer;
use typst::{Config, World};
@ -55,7 +54,7 @@ fn bench_tokenize(iai: &mut Iai) {
}
fn bench_parse(iai: &mut Iai) {
iai.run(|| typst::parse::parse(TEXT));
iai.run(|| typst::syntax::parse(TEXT));
}
fn bench_edit(iai: &mut Iai) {
@ -77,15 +76,15 @@ fn bench_highlight(iai: &mut Iai) {
fn bench_eval(iai: &mut Iai) {
let world = BenchWorld::new();
let id = world.source.id();
let route = typst::eval::Route::default();
iai.run(|| typst::eval::eval(world.track(), route.track(), id).unwrap());
let route = typst::model::Route::default();
iai.run(|| typst::model::eval(world.track(), route.track(), id).unwrap());
}
fn bench_layout(iai: &mut Iai) {
let world = BenchWorld::new();
let id = world.source.id();
let route = typst::eval::Route::default();
let module = typst::eval::eval(world.track(), route.track(), id).unwrap();
let route = typst::model::Route::default();
let module = typst::model::eval(world.track(), route.track(), id).unwrap();
iai.run(|| typst::model::layout(world.track(), &module.content));
}

4
macros/src/lib.rs
View File

@ -69,12 +69,12 @@ fn expand(stream: TokenStream2, mut impl_block: syn::ItemImpl) -> Result<TokenSt
use std::any::TypeId;
use std::marker::PhantomData;
use once_cell::sync::Lazy;
use crate::{eval, model};
use crate::model;
use super::*;
#impl_block
impl<#params> eval::Node for #self_ty {
impl<#params> model::Node for #self_ty {
const SHOWABLE: bool = #showable;
#construct
#set

2
src/frame.rs
View File

@ -4,13 +4,13 @@ use std::fmt::{self, Debug, Formatter, Write};
use std::num::NonZeroUsize;
use std::sync::Arc;
use crate::eval::{Dict, Value};
use crate::font::Font;
use crate::geom::{
Align, Em, Length, Numeric, Paint, Point, Shape, Size, Spec, Transform,
};
use crate::image::Image;
use crate::library::text::Lang;
use crate::model::{Dict, Value};
use crate::util::EcoString;
/// A finished layout with elements at fixed positions.

19
src/lib.rs
View File

@ -17,12 +17,12 @@
//! - **Exporting:** The finished layout can be exported into a supported
//! format. Currently, the only supported output format is [PDF].
//!
//! [tokens]: parse::Tokens
//! [parsed]: parse::parse
//! [tokens]: syntax::Tokens
//! [parsed]: syntax::parse
//! [syntax tree]: syntax::SyntaxNode
//! [AST]: syntax::ast
//! [evaluate]: eval::eval
//! [module]: eval::Module
//! [evaluate]: model::eval
//! [module]: model::Module
//! [content]: model::Content
//! [layouted]: model::layout
//! [PDF]: export::pdf
@ -38,15 +38,12 @@ pub mod geom;
#[macro_use]
pub mod diag;
#[macro_use]
pub mod eval;
pub mod model;
pub mod export;
pub mod font;
pub mod frame;
pub mod image;
pub mod library;
pub mod model;
pub mod parse;
pub mod source;
pub mod syntax;
use std::path::{Path, PathBuf};
@ -54,11 +51,11 @@ use std::path::{Path, PathBuf};
use comemo::{Prehashed, Track};
use crate::diag::{FileResult, SourceResult};
use crate::eval::{Route, Scope};
use crate::font::{Font, FontBook};
use crate::frame::Frame;
use crate::model::StyleMap;
use crate::source::{Source, SourceId};
use crate::model::{Route, Scope};
use crate::syntax::{Source, SourceId};
use crate::util::Buffer;
/// Typeset a source file into a collection of layouted frames.
@ -71,7 +68,7 @@ pub fn typeset(
main: SourceId,
) -> SourceResult<Vec<Frame>> {
let route = Route::default();
let module = eval::eval(world.track(), route.track(), main)?;
let module = model::eval(world.track(), route.track(), main)?;
model::layout(world.track(), &module.content)
}

6
src/library/prelude.rs
View File

@ -12,12 +12,12 @@ pub use typst_macros::node;
pub use crate::diag::{
with_alternative, At, FileError, FileResult, SourceError, SourceResult, StrResult,
};
pub use crate::eval::{
pub use crate::frame::*;
pub use crate::geom::*;
pub use crate::model::{
Arg, Args, Array, Cast, Dict, Dynamic, Func, Node, RawAlign, RawLength, RawStroke,
Scope, Smart, Str, Value, Vm,
};
pub use crate::frame::*;
pub use crate::geom::*;
pub use crate::model::{
Content, Fold, Key, Layout, LayoutNode, Regions, Resolve, Selector, Show, ShowNode,
StyleChain, StyleMap, StyleVec,

2
src/library/text/lang.rs
View File

@ -1,5 +1,5 @@
use crate::eval::Value;
use crate::geom::Dir;
use crate::model::Value;
/// A code for a natural language.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]

2
src/library/text/quotes.rs
View File

@ -1,5 +1,5 @@
use super::{Lang, Region};
use crate::parse::is_newline;
use crate::syntax::is_newline;
/// State machine for smart quote subtitution.
#[derive(Debug, Clone)]

4
src/library/text/raw.rs
View File

@ -72,8 +72,8 @@ impl Show for RawNode {
let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) {
let root = match lang.as_deref() {
Some("typc") => crate::parse::parse_code(&self.text),
_ => crate::parse::parse(&self.text),
Some("typc") => crate::syntax::parse_code(&self.text),
_ => crate::syntax::parse(&self.text),
};
let mut seq = vec![];

4
src/library/utility/mod.rs
View File

@ -12,9 +12,9 @@ pub use string::*;
use comemo::Track;
use crate::eval::{Eval, Route, Scopes, Vm};
use crate::library::prelude::*;
use crate::source::Source;
use crate::model::{Eval, Route, Scopes, Vm};
use crate::syntax::Source;
/// The name of a value's type.
pub fn type_(_: &mut Vm, args: &mut Args) -> SourceResult<Value> {

2
src/library/utility/string.rs
View File

@ -1,5 +1,5 @@
use crate::eval::Regex;
use crate::library::prelude::*;
use crate::model::Regex;
/// The string representation of a value.
pub fn repr(_: &mut Vm, args: &mut Args) -> SourceResult<Value> {

2
src/main.rs
View File

@ -21,7 +21,7 @@ use walkdir::WalkDir;
use typst::diag::{FileError, FileResult, SourceError, StrResult};
use typst::font::{Font, FontBook, FontInfo, FontVariant};
use typst::source::{Source, SourceId};
use typst::syntax::{Source, SourceId};
use typst::util::{Buffer, PathExt};
use typst::{Config, World};

0
src/eval/args.rs → src/model/args.rs
View File

4
src/eval/array.rs → src/model/array.rs
View File

@ -12,11 +12,11 @@ use crate::util::ArcExt;
#[allow(unused_macros)]
macro_rules! array {
($value:expr; $count:expr) => {
$crate::eval::Array::from_vec(vec![$value.into(); $count])
$crate::model::Array::from_vec(vec![$value.into(); $count])
};
($($value:expr),* $(,)?) => {
$crate::eval::Array::from_vec(vec![$($value.into()),*])
$crate::model::Array::from_vec(vec![$($value.into()),*])
};
}

2
src/eval/capture.rs → src/model/capture.rs
View File

@ -133,7 +133,7 @@ impl<'a> CapturesVisitor<'a> {
#[cfg(test)]
mod tests {
use super::*;
use crate::parse::parse;
use crate::syntax::parse;
#[track_caller]
fn test(text: &str, result: &[&str]) {

27
src/eval/cast.rs → src/model/cast.rs
View File

@ -1,9 +1,8 @@
use std::num::NonZeroUsize;
use super::{Regex, Value};
use super::{Content, Layout, LayoutNode, Pattern, Regex, Value};
use crate::diag::{with_alternative, StrResult};
use crate::geom::{Corners, Dir, Paint, Sides};
use crate::model::{Content, Layout, LayoutNode, Pattern};
use crate::syntax::Spanned;
use crate::util::EcoString;
@ -19,20 +18,20 @@ pub trait Cast<V = Value>: Sized {
/// Implement traits for dynamic types.
macro_rules! dynamic {
($type:ty: $name:literal, $($tts:tt)*) => {
impl $crate::eval::Type for $type {
impl $crate::model::Type for $type {
const TYPE_NAME: &'static str = $name;
}
castable! {
$type,
Expected: <Self as $crate::eval::Type>::TYPE_NAME,
Expected: <Self as $crate::model::Type>::TYPE_NAME,
$($tts)*
@this: Self => this.clone(),
}
impl From<$type> for $crate::eval::Value {
impl From<$type> for $crate::model::Value {
fn from(v: $type) -> Self {
$crate::eval::Value::Dyn($crate::eval::Dynamic::new(v))
$crate::model::Value::Dyn($crate::model::Dynamic::new(v))
}
}
};
@ -41,12 +40,12 @@ macro_rules! dynamic {
/// Make a type castable from a value.
macro_rules! castable {
($type:ty: $inner:ty) => {
impl $crate::eval::Cast<$crate::eval::Value> for $type {
fn is(value: &$crate::eval::Value) -> bool {
impl $crate::model::Cast<$crate::model::Value> for $type {
fn is(value: &$crate::model::Value) -> bool {
<$inner>::is(value)
}
fn cast(value: $crate::eval::Value) -> $crate::diag::StrResult<Self> {
fn cast(value: $crate::model::Value) -> $crate::diag::StrResult<Self> {
<$inner>::cast(value).map(Self)
}
}
@ -59,22 +58,22 @@ macro_rules! castable {
$(@$dyn_in:ident: $dyn_type:ty => $dyn_out:expr,)*
) => {
#[allow(unreachable_patterns)]
impl $crate::eval::Cast<$crate::eval::Value> for $type {
fn is(value: &$crate::eval::Value) -> bool {
impl $crate::model::Cast<$crate::model::Value> for $type {
fn is(value: &$crate::model::Value) -> bool {
#[allow(unused_variables)]
match value {
$($pattern => true,)*
$crate::eval::Value::Dyn(dynamic) => {
$crate::model::Value::Dyn(dynamic) => {
false $(|| dynamic.is::<$dyn_type>())*
}
_ => false,
}
}
fn cast(value: $crate::eval::Value) -> $crate::diag::StrResult<Self> {
fn cast(value: $crate::model::Value) -> $crate::diag::StrResult<Self> {
let found = match value {
$($pattern => return Ok($out),)*
$crate::eval::Value::Dyn(dynamic) => {
$crate::model::Value::Dyn(dynamic) => {
$(if let Some($dyn_in) = dynamic.downcast::<$dyn_type>() {
return Ok($dyn_out);
})*

31
src/model/content.rs
View File

@ -21,21 +21,6 @@ use crate::library::text::{
use crate::util::EcoString;
use crate::World;
/// Layout content into a collection of pages.
///
/// Relayouts until all pinned locations are converged.
#[comemo::memoize]
pub fn layout(world: Tracked<dyn World>, content: &Content) -> SourceResult<Vec<Frame>> {
let styles = StyleChain::with_root(&world.config().styles);
let scratch = Scratch::default();
let mut builder = Builder::new(world, &scratch, true);
builder.accept(content, styles)?;
let (doc, shared) = builder.into_doc(styles)?;
doc.layout(world, shared)
}
/// Composable representation of styled content.
///
/// This results from:
@ -332,7 +317,7 @@ impl Sum for Content {
}
/// Builds a document or a flow node from content.
struct Builder<'a> {
pub(super) struct Builder<'a> {
/// The core context.
world: Tracked<'a, dyn World>,
/// Scratch arenas for building.
@ -349,7 +334,7 @@ struct Builder<'a> {
/// Temporary storage arenas for building.
#[derive(Default)]
struct Scratch<'a> {
pub(super) struct Scratch<'a> {
/// An arena where intermediate style chains are stored.
styles: Arena<StyleChain<'a>>,
/// An arena where intermediate content resulting from show rules is stored.
@ -357,7 +342,11 @@ struct Scratch<'a> {
}
impl<'a> Builder<'a> {
fn new(world: Tracked<'a, dyn World>, scratch: &'a Scratch<'a>, top: bool) -> Self {
pub fn new(
world: Tracked<'a, dyn World>,
scratch: &'a Scratch<'a>,
top: bool,
) -> Self {
Self {
world,
scratch,
@ -368,7 +357,7 @@ impl<'a> Builder<'a> {
}
}
fn into_doc(
pub fn into_doc(
mut self,
styles: StyleChain<'a>,
) -> SourceResult<(DocNode, StyleChain<'a>)> {
@ -377,7 +366,7 @@ impl<'a> Builder<'a> {
Ok((DocNode(pages), shared))
}
fn into_flow(
pub fn into_flow(
mut self,
styles: StyleChain<'a>,
) -> SourceResult<(FlowNode, StyleChain<'a>)> {
@ -386,7 +375,7 @@ impl<'a> Builder<'a> {
Ok((FlowNode(children), shared))
}
fn accept(
pub fn accept(
&mut self,
content: &'a Content,
styles: StyleChain<'a>,

4
src/eval/dict.rs → src/model/dict.rs
View File

@ -5,7 +5,7 @@ use std::sync::Arc;
use super::{Args, Array, Func, Str, Value, Vm};
use crate::diag::{SourceResult, StrResult};
use crate::parse::is_ident;
use crate::syntax::is_ident;
use crate::syntax::Spanned;
use crate::util::ArcExt;
@ -16,7 +16,7 @@ macro_rules! dict {
#[allow(unused_mut)]
let mut map = std::collections::BTreeMap::new();
$(map.insert($key.into(), $value.into());)*
$crate::eval::Dict::from_map(map)
$crate::model::Dict::from_map(map)
}};
}

40
src/eval/mod.rs → src/model/eval.rs
View File

@ -1,50 +1,20 @@
//! Evaluation of markup into modules.
#[macro_use]
mod cast;
#[macro_use]
mod array;
#[macro_use]
mod dict;
#[macro_use]
mod str;
#[macro_use]
mod value;
mod args;
mod capture;
mod func;
pub mod methods;
pub mod ops;
mod raw;
mod scope;
mod vm;
pub use self::str::*;
pub use args::*;
pub use array::*;
pub use capture::*;
pub use cast::*;
pub use dict::*;
pub use func::*;
pub use raw::*;
pub use scope::*;
pub use typst_macros::node;
pub use value::*;
pub use vm::*;
use std::collections::BTreeMap;
use std::sync::Arc;
use comemo::{Track, Tracked};
use unicode_segmentation::UnicodeSegmentation;
use super::{
methods, ops, Arg, Args, Array, CapturesVisitor, Closure, Content, Dict, Flow, Func,
Pattern, Recipe, Scope, Scopes, StyleEntry, StyleMap, Value, Vm,
};
use crate::diag::{At, SourceResult, StrResult, Trace, Tracepoint};
use crate::geom::{Angle, Em, Fraction, Length, Ratio};
use crate::library;
use crate::model::{Content, Pattern, Recipe, StyleEntry, StyleMap};
use crate::source::SourceId;
use crate::syntax::ast::TypedNode;
use crate::syntax::{ast, Span, Spanned, Unit};
use crate::syntax::{ast, SourceId, Span, Spanned, Unit};
use crate::util::EcoString;
use crate::World;

7
src/eval/func.rs → src/model/func.rs
View File

@ -4,11 +4,12 @@ use std::sync::Arc;
use comemo::{Track, Tracked};
use super::{Args, Eval, Flow, Route, Scope, Scopes, Value, Vm};
use super::{
Args, Content, Eval, Flow, NodeId, Route, Scope, Scopes, StyleMap, Value, Vm,
};
use crate::diag::{SourceResult, StrResult};
use crate::model::{Content, NodeId, StyleMap};
use crate::source::SourceId;
use crate::syntax::ast::Expr;
use crate::syntax::SourceId;
use crate::util::EcoString;
use crate::World;

17
src/model/layout.rs
View File

@ -8,8 +8,8 @@ use std::sync::Arc;
use comemo::{Prehashed, Tracked};
use super::{Barrier, NodeId, Resolve, StyleChain, StyleEntry};
use super::{Builder, Content, RawAlign, RawLength, Scratch};
use crate::diag::SourceResult;
use crate::eval::{RawAlign, RawLength};
use crate::frame::{Element, Frame};
use crate::geom::{
Align, Geometry, Length, Paint, Point, Relative, Sides, Size, Spec, Stroke,
@ -18,6 +18,21 @@ use crate::library::graphics::MoveNode;
use crate::library::layout::{AlignNode, PadNode};
use crate::World;
/// Layout content into a collection of pages.
///
/// Relayouts until all pinned locations are converged.
#[comemo::memoize]
pub fn layout(world: Tracked<dyn World>, content: &Content) -> SourceResult<Vec<Frame>> {
let styles = StyleChain::with_root(&world.config().styles);
let scratch = Scratch::default();
let mut builder = Builder::new(world, &scratch, true);
builder.accept(content, styles)?;
let (doc, shared) = builder.into_doc(styles)?;
doc.layout(world, shared)
}
/// A node that can be layouted into a sequence of regions.
///
/// Layouting returns one frame per used region.

0
src/eval/methods.rs → src/model/methods.rs
View File

34
src/model/mod.rs
View File

@ -1,18 +1,50 @@
//! Styled and structured representation of layoutable content.
//! Layout and computation model.
#[macro_use]
mod styles;
mod collapse;
mod content;
mod eval;
mod layout;
mod property;
mod recipe;
mod show;
#[macro_use]
mod cast;
#[macro_use]
mod array;
#[macro_use]
mod dict;
#[macro_use]
mod str;
#[macro_use]
mod value;
mod args;
mod capture;
mod func;
pub mod methods;
pub mod ops;
mod raw;
mod scope;
mod vm;
pub use self::str::*;
pub use args::*;
pub use array::*;
pub use capture::*;
pub use cast::*;
pub use collapse::*;
pub use content::*;
pub use dict::*;
pub use eval::*;
pub use func::*;
pub use layout::*;
pub use property::*;
pub use raw::*;
pub use recipe::*;
pub use scope::*;
pub use show::*;
pub use styles::*;
pub use typst_macros::node;
pub use value::*;
pub use vm::*;

9
src/eval/ops.rs → src/model/ops.rs
View File

@ -5,7 +5,6 @@ use std::cmp::Ordering;
use super::{RawAlign, RawLength, RawStroke, Regex, Smart, Value};
use crate::diag::StrResult;
use crate::geom::{Numeric, Relative, Spec, SpecAxis};
use crate::model;
use Value::*;
/// Bail with a type mismatch error.
@ -21,8 +20,8 @@ pub fn join(lhs: Value, rhs: Value) -> StrResult<Value> {
(a, None) => a,
(None, b) => b,
(Str(a), Str(b)) => Str(a + b),
(Str(a), Content(b)) => Content(model::Content::Text(a.into()) + b),
(Content(a), Str(b)) => Content(a + model::Content::Text(b.into())),
(Str(a), Content(b)) => Content(super::Content::Text(a.into()) + b),
(Content(a), Str(b)) => Content(a + super::Content::Text(b.into())),
(Content(a), Content(b)) => Content(a + b),
(Array(a), Array(b)) => Array(a + b),
(Dict(a), Dict(b)) => Dict(a + b),
@ -87,8 +86,8 @@ pub fn add(lhs: Value, rhs: Value) -> StrResult<Value> {
(Str(a), Str(b)) => Str(a + b),
(Content(a), Content(b)) => Content(a + b),
(Content(a), Str(b)) => Content(a + model::Content::Text(b.into())),
(Str(a), Content(b)) => Content(model::Content::Text(a.into()) + b),
(Content(a), Str(b)) => Content(a + super::Content::Text(b.into())),
(Str(a), Content(b)) => Content(super::Content::Text(a.into()) + b),
(Array(a), Array(b)) => Array(a + b),
(Dict(a), Dict(b)) => Dict(a + b),

3
src/model/property.rs
View File

@ -5,8 +5,7 @@ use std::sync::Arc;
use comemo::Prehashed;
use super::{Interruption, NodeId, StyleChain};
use crate::eval::{RawLength, Smart};
use super::{Interruption, NodeId, RawLength, Smart, StyleChain};
use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec};
use crate::library::layout::PageNode;
use crate::library::structure::{DescNode, EnumNode, ListNode};

3
src/eval/raw.rs → src/model/raw.rs
View File

@ -2,12 +2,11 @@ use std::cmp::Ordering;
use std::fmt::{self, Debug, Formatter};
use std::ops::{Add, Div, Mul, Neg};
use super::{Smart, Value};
use super::{Fold, Resolve, Smart, StyleChain, Value};
use crate::geom::{
Align, Em, Get, Length, Numeric, Paint, Relative, Spec, SpecAxis, Stroke,
};
use crate::library::text::TextNode;
use crate::model::{Fold, Resolve, StyleChain};
/// The unresolved alignment representation.
#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]

6
src/model/recipe.rs
View File

@ -2,9 +2,11 @@ use std::fmt::{self, Debug, Formatter};
use comemo::Tracked;
use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry};
use super::{
Args, Content, Func, Interruption, NodeId, Regex, Show, ShowNode, StyleChain,
StyleEntry, Value,
};
use crate::diag::SourceResult;
use crate::eval::{Args, Func, Regex, Value};
use crate::library::structure::{DescNode, EnumNode, ListNode};
use crate::syntax::Spanned;
use crate::World;

0
src/eval/scope.rs → src/model/scope.rs
View File

3
src/model/show.rs
View File

@ -4,9 +4,8 @@ use std::sync::Arc;
use comemo::{Prehashed, Tracked};
use super::{Content, NodeId, Selector, StyleChain};
use super::{Content, Dict, NodeId, Selector, StyleChain};
use crate::diag::SourceResult;
use crate::eval::Dict;
use crate::World;
/// A node that can be realized given some styles.

85
src/eval/str.rs → src/model/str.rs
View File

@ -13,7 +13,7 @@ use crate::util::EcoString;
#[allow(unused_macros)]
macro_rules! format_str {
($($tts:tt)*) => {{
$crate::eval::Str::from(format_eco!($($tts)*))
$crate::model::Str::from(format_eco!($($tts)*))
}};
}
@ -76,69 +76,69 @@ impl Str {
}
/// Whether the given pattern exists in this string.
pub fn contains(&self, pattern: TextPattern) -> bool {
pub fn contains(&self, pattern: StrPattern) -> bool {
match pattern {
TextPattern::Str(pat) => self.0.contains(pat.as_str()),
TextPattern::Regex(re) => re.is_match(self),
StrPattern::Str(pat) => self.0.contains(pat.as_str()),
StrPattern::Regex(re) => re.is_match(self),
}
}
/// Whether this string begins with the given pattern.
pub fn starts_with(&self, pattern: TextPattern) -> bool {
pub fn starts_with(&self, pattern: StrPattern) -> bool {
match pattern {
TextPattern::Str(pat) => self.0.starts_with(pat.as_str()),
TextPattern::Regex(re) => re.find(self).map_or(false, |m| m.start() == 0),
StrPattern::Str(pat) => self.0.starts_with(pat.as_str()),
StrPattern::Regex(re) => re.find(self).map_or(false, |m| m.start() == 0),
}
}
/// Whether this string ends with the given pattern.
pub fn ends_with(&self, pattern: TextPattern) -> bool {
pub fn ends_with(&self, pattern: StrPattern) -> bool {
match pattern {
TextPattern::Str(pat) => self.0.ends_with(pat.as_str()),
TextPattern::Regex(re) => {
StrPattern::Str(pat) => self.0.ends_with(pat.as_str()),
StrPattern::Regex(re) => {
re.find_iter(self).last().map_or(false, |m| m.end() == self.0.len())
}
}
}
/// The text of the pattern's first match in this string.
pub fn find(&self, pattern: TextPattern) -> Option<Self> {
pub fn find(&self, pattern: StrPattern) -> Option<Self> {
match pattern {
TextPattern::Str(pat) => self.0.contains(pat.as_str()).then(|| pat),
TextPattern::Regex(re) => re.find(self).map(|m| m.as_str().into()),
StrPattern::Str(pat) => self.0.contains(pat.as_str()).then(|| pat),
StrPattern::Regex(re) => re.find(self).map(|m| m.as_str().into()),
}
}
/// The position of the pattern's first match in this string.
pub fn position(&self, pattern: TextPattern) -> Option<i64> {
pub fn position(&self, pattern: StrPattern) -> Option<i64> {
match pattern {
TextPattern::Str(pat) => self.0.find(pat.as_str()).map(|i| i as i64),
TextPattern::Regex(re) => re.find(self).map(|m| m.start() as i64),
StrPattern::Str(pat) => self.0.find(pat.as_str()).map(|i| i as i64),
StrPattern::Regex(re) => re.find(self).map(|m| m.start() as i64),
}
}
/// The start and, text and capture groups (if any) of the first match of
/// the pattern in this string.
pub fn match_(&self, pattern: TextPattern) -> Option<Dict> {
pub fn match_(&self, pattern: StrPattern) -> Option<Dict> {
match pattern {
TextPattern::Str(pat) => {
StrPattern::Str(pat) => {
self.0.match_indices(pat.as_str()).next().map(match_to_dict)
}
TextPattern::Regex(re) => re.captures(self).map(captures_to_dict),
StrPattern::Regex(re) => re.captures(self).map(captures_to_dict),
}
}
/// The start, end, text and capture groups (if any) of all matches of the
/// pattern in this string.
pub fn matches(&self, pattern: TextPattern) -> Array {
pub fn matches(&self, pattern: StrPattern) -> Array {
match pattern {
TextPattern::Str(pat) => self
StrPattern::Str(pat) => self
.0
.match_indices(pat.as_str())
.map(match_to_dict)
.map(Value::Dict)
.collect(),
TextPattern::Regex(re) => re
StrPattern::Regex(re) => re
.captures_iter(self)
.map(captures_to_dict)
.map(Value::Dict)
@ -147,14 +147,14 @@ impl Str {
}
/// Split this string at whitespace or a specific pattern.
pub fn split(&self, pattern: Option<TextPattern>) -> Array {
pub fn split(&self, pattern: Option<StrPattern>) -> Array {
let s = self.as_str();
match pattern {
None => s.split_whitespace().map(|v| Value::Str(v.into())).collect(),
Some(TextPattern::Str(pat)) => {
Some(StrPattern::Str(pat)) => {
s.split(pat.as_str()).map(|v| Value::Str(v.into())).collect()
}
Some(TextPattern::Regex(re)) => {
Some(StrPattern::Regex(re)) => {
re.split(s).map(|v| Value::Str(v.into())).collect()
}
}
@ -166,20 +166,20 @@ impl Str {
/// pattern.
pub fn trim(
&self,
pattern: Option<TextPattern>,
at: Option<TextSide>,
pattern: Option<StrPattern>,
at: Option<StrSide>,
repeat: bool,
) -> Self {
let mut start = matches!(at, Some(TextSide::Start) | None);
let end = matches!(at, Some(TextSide::End) | None);
let mut start = matches!(at, Some(StrSide::Start) | None);
let end = matches!(at, Some(StrSide::End) | None);
let trimmed = match pattern {
None => match at {
None => self.0.trim(),
Some(TextSide::Start) => self.0.trim_start(),
Some(TextSide::End) => self.0.trim_end(),
Some(StrSide::Start) => self.0.trim_start(),
Some(StrSide::End) => self.0.trim_end(),
},
Some(TextPattern::Str(pat)) => {
Some(StrPattern::Str(pat)) => {
let pat = pat.as_str();
let mut s = self.as_str();
if repeat {
@ -199,7 +199,7 @@ impl Str {
}
s
}
Some(TextPattern::Regex(re)) => {
Some(StrPattern::Regex(re)) => {
let s = self.as_str();
let mut last = 0;
let mut range = 0 .. s.len();
@ -239,18 +239,13 @@ impl Str {
/// Replace at most `count` occurances of the given pattern with a
/// replacement string (beginning from the start).
pub fn replace(
&self,
pattern: TextPattern,
with: Self,
count: Option<usize>,
) -> Self {
pub fn replace(&self, pattern: StrPattern, with: Self, count: Option<usize>) -> Self {
match pattern {
TextPattern::Str(pat) => match count {
StrPattern::Str(pat) => match count {
Some(n) => self.0.replacen(pat.as_str(), &with, n).into(),
None => self.0.replace(pat.as_str(), &with).into(),
},
TextPattern::Regex(re) => match count {
StrPattern::Regex(re) => match count {
Some(n) => re.replacen(self, n, with.as_str()).into(),
None => re.replace(self, with.as_str()).into(),
},
@ -440,7 +435,7 @@ impl Hash for Regex {
/// A pattern which can be searched for in a string.
#[derive(Debug, Clone)]
pub enum TextPattern {
pub enum StrPattern {
/// Just a string.
Str(Str),
/// A regular expression.
@ -448,7 +443,7 @@ pub enum TextPattern {
}
castable! {
TextPattern,
StrPattern,
Expected: "string or regular expression",
Value::Str(text) => Self::Str(text),
@regex: Regex => Self::Regex(regex.clone()),
@ -456,7 +451,7 @@ castable! {
/// A side of a string.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub enum TextSide {
pub enum StrSide {
/// The logical start of the string, may be left or right depending on the
/// language.
Start,
@ -465,7 +460,7 @@ pub enum TextSide {
}
castable! {
TextSide,
StrSide,
Expected: "start or end",
@align: RawAlign => match align {
RawAlign::Start => Self::Start,

2
src/model/styles.rs
View File

@ -107,7 +107,7 @@ impl StyleMap {
/// Mark all contained properties as _scoped_. This means that they only
/// apply to the first descendant node (of their type) in the hierarchy and
/// not its children, too. This is used by
/// [constructors](crate::eval::Node::construct).
/// [constructors](super::Node::construct).
pub fn scoped(mut self) -> Self {
for entry in &mut self.0 {
if let StyleEntry::Property(property) = entry {

15
src/eval/value.rs → src/model/value.rs
View File

@ -4,11 +4,12 @@ use std::fmt::{self, Debug, Formatter};
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use super::{ops, Args, Array, Cast, Dict, Func, RawLength, Str};
use siphasher::sip128::{Hasher128, SipHasher};
use super::{ops, Args, Array, Cast, Content, Dict, Func, Layout, RawLength, Str};
use crate::diag::StrResult;
use crate::geom::{Angle, Color, Em, Fraction, Length, Ratio, Relative, RgbaColor};
use crate::library::text::RawNode;
use crate::model::{Content, Layout};
use crate::util::EcoString;
/// A computational value.
@ -296,7 +297,7 @@ trait Bounds: Debug + Sync + Send + 'static {
fn as_any(&self) -> &dyn Any;
fn dyn_eq(&self, other: &Dynamic) -> bool;
fn dyn_type_name(&self) -> &'static str;
fn hash64(&self) -> u64;
fn hash128(&self) -> u128;
}
impl<T> Bounds for T
@ -319,19 +320,19 @@ where
T::TYPE_NAME
}
fn hash64(&self) -> u64 {
fn hash128(&self) -> u128 {
// Also hash the TypeId since nodes with different types but
// equal data should be different.
let mut state = fxhash::FxHasher64::default();
let mut state = SipHasher::new();
self.type_id().hash(&mut state);
self.hash(&mut state);
state.finish()
state.finish128().as_u128()
}
}
impl Hash for dyn Bounds {
fn hash<H: Hasher>(&self, state: &mut H) {
state.write_u64(self.hash64());
state.write_u128(self.hash128());
}
}

3
src/eval/vm.rs → src/model/vm.rs
View File

@ -4,8 +4,7 @@ use comemo::Tracked;
use super::{Route, Scopes, Value};
use crate::diag::{SourceError, StrResult};
use crate::source::SourceId;
use crate::syntax::Span;
use crate::syntax::{SourceId, Span};
use crate::util::PathExt;
use crate::World;

6
src/syntax/highlight.rs
View File

@ -6,7 +6,7 @@ use std::ops::Range;
use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme};
use syntect::parsing::Scope;
use super::{NodeKind, SyntaxNode};
use super::{parse, NodeKind, SyntaxNode};
/// Highlight source text into a standalone HTML document.
pub fn highlight_html(text: &str, theme: &Theme) -> String {
@ -28,7 +28,7 @@ pub fn highlight_pre(text: &str, theme: &Theme) -> String {
let mut buf = String::new();
buf.push_str("<pre>\n");
let root = crate::parse::parse(text);
let root = parse(text);
highlight_themed(&root, theme, |range, style| {
let styled = style != Style::default();
if styled {
@ -401,8 +401,8 @@ impl Category {
#[cfg(test)]
mod tests {
use super::super::Source;
use super::*;
use crate::source::Source;
#[test]
fn test_highlighting() {

10
src/parse/incremental.rs → src/syntax/incremental.rs
View File

@ -1,10 +1,9 @@
use std::ops::Range;
use std::sync::Arc;
use crate::syntax::{InnerNode, NodeKind, Span, SyntaxNode};
use super::{
is_newline, parse, reparse_code_block, reparse_content_block, reparse_markup_elements,
is_newline, parse, reparse_code_block, reparse_content_block,
reparse_markup_elements, InnerNode, NodeKind, Span, SyntaxNode,
};
/// Refresh the given syntax node with as little parsing as possible.
@ -413,9 +412,8 @@ fn next_at_start(kind: &NodeKind, prev: bool) -> bool {
#[rustfmt::skip]
mod tests {
use super::*;
use crate::parse::parse;
use crate::parse::tests::check;
use crate::source::Source;
use super::super::{parse, Source};
use super::super::tests::check;
#[track_caller]
fn test(prev: &str, range: Range<usize>, with: &str, goal: Range<usize>) {

567
src/syntax/mod.rs
View File

@ -1,558 +1,41 @@
//! Syntax types.
//! Syntax definition, parsing, and highlighting.
pub mod ast;
pub mod highlight;
mod incremental;
mod kind;
mod node;
mod parser;
mod parsing;
mod resolve;
mod source;
mod span;
use std::fmt::{self, Debug, Formatter};
use std::ops::Range;
use std::sync::Arc;
mod tokens;
pub use kind::*;
pub use node::*;
pub use parsing::*;
pub use source::*;
pub use span::*;
pub use tokens::*;
use self::ast::TypedNode;
use crate::diag::SourceError;
use crate::source::SourceId;
use incremental::reparse;
use parser::*;
/// An inner or leaf node in the untyped syntax tree.
#[derive(Clone, PartialEq, Hash)]
pub enum SyntaxNode {
/// A reference-counted inner node.
Inner(Arc<InnerNode>),
/// A leaf token.
Leaf(NodeData),
}
#[cfg(test)]
mod tests {
use std::fmt::Debug;
impl SyntaxNode {
/// The metadata of the node.
pub fn data(&self) -> &NodeData {
match self {
Self::Inner(inner) => &inner.data,
Self::Leaf(leaf) => leaf,
}
}
/// The type of the node.
pub fn kind(&self) -> &NodeKind {
self.data().kind()
}
/// The length of the node.
pub fn len(&self) -> usize {
self.data().len()
}
/// The number of descendants, including the node itself.
pub fn descendants(&self) -> usize {
match self {
Self::Inner(inner) => inner.descendants(),
Self::Leaf(_) => 1,
}
}
/// The span of the node.
pub fn span(&self) -> Span {
self.data().span()
}
/// Whether the node or its children contain an error.
pub fn erroneous(&self) -> bool {
match self {
Self::Inner(node) => node.erroneous,
Self::Leaf(data) => data.kind.is_error(),
}
}
/// The error messages for this node and its descendants.
pub fn errors(&self) -> Vec<SourceError> {
if !self.erroneous() {
return vec![];
}
match self.kind() {
NodeKind::Error(pos, message) => {
vec![SourceError::new(self.span(), message.clone()).with_pos(*pos)]
}
_ => self
.children()
.filter(|node| node.erroneous())
.flat_map(|node| node.errors())
.collect(),
}
}
/// The node's children.
pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> {
match self {
Self::Inner(inner) => inner.children(),
Self::Leaf(_) => [].iter(),
}
}
/// Convert the node to a typed AST node.
pub fn cast<T>(&self) -> Option<T>
#[track_caller]
pub fn check<T>(text: &str, found: T, expected: T)
where
T: TypedNode,
T: Debug + PartialEq,
{
T::from_untyped(self)
}
/// Get the first child that can cast to the AST type `T`.
pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> {
self.children().find_map(Self::cast)
}
/// Get the last child that can cast to the AST type `T`.
pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> {
self.children().rev().find_map(Self::cast)
}
/// Change the type of the node.
pub fn convert(&mut self, kind: NodeKind) {
match self {
Self::Inner(inner) => {
let node = Arc::make_mut(inner);
node.erroneous |= kind.is_error();
node.data.kind = kind;
}
Self::Leaf(leaf) => leaf.kind = kind,
}
}
/// Set a synthetic span for the node and all its descendants.
pub fn synthesize(&mut self, span: Span) {
match self {
Self::Inner(inner) => Arc::make_mut(inner).synthesize(span),
Self::Leaf(leaf) => leaf.synthesize(span),
}
}
/// Assign spans to each node.
pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult {
match self {
Self::Inner(inner) => Arc::make_mut(inner).numberize(id, None, within),
Self::Leaf(leaf) => leaf.numberize(id, within),
}
}
/// The upper bound of assigned numbers in this subtree.
pub fn upper(&self) -> u64 {
match self {
Self::Inner(inner) => inner.upper(),
Self::Leaf(leaf) => leaf.span().number() + 1,
}
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> {
match self {
Self::Inner(inner) => inner.range(span, offset),
Self::Leaf(leaf) => leaf.range(span, offset),
}
}
/// Returns all leaf descendants of this node (may include itself).
///
/// This method is slow and only intended for testing.
pub fn leafs(&self) -> Vec<Self> {
if match self {
Self::Inner(inner) => inner.children.is_empty(),
Self::Leaf(_) => true,
} {
vec![self.clone()]
} else {
self.children().flat_map(Self::leafs).collect()
if found != expected {
println!("source: {text:?}");
println!("expected: {expected:#?}");
println!("found: {found:#?}");
panic!("test failed");
}
}
}
impl Default for SyntaxNode {
fn default() -> Self {
Self::Leaf(NodeData::new(NodeKind::None, 0))
}
}
impl Debug for SyntaxNode {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Self::Inner(node) => node.fmt(f),
Self::Leaf(token) => token.fmt(f),
}
}
}
/// An inner node in the untyped syntax tree.
#[derive(Clone, Hash)]
pub struct InnerNode {
/// Node metadata.
data: NodeData,
/// The number of nodes in the whole subtree, including this node.
descendants: usize,
/// Whether this node or any of its children are erroneous.
erroneous: bool,
/// The upper bound of this node's numbering range.
upper: u64,
/// This node's children, losslessly make up this node.
children: Vec<SyntaxNode>,
}
impl InnerNode {
/// Creates a new node with the given kind and a single child.
pub fn with_child(kind: NodeKind, child: impl Into<SyntaxNode>) -> Self {
Self::with_children(kind, vec![child.into()])
}
/// Creates a new node with the given kind and children.
pub fn with_children(kind: NodeKind, children: Vec<SyntaxNode>) -> Self {
let mut len = 0;
let mut descendants = 1;
let mut erroneous = kind.is_error();
for child in &children {
len += child.len();
descendants += child.descendants();
erroneous |= child.erroneous();
}
Self {
data: NodeData::new(kind, len),
descendants,
erroneous,
upper: 0,
children,
}
}
/// The node's metadata.
pub fn data(&self) -> &NodeData {
&self.data
}
/// The node's type.
pub fn kind(&self) -> &NodeKind {
self.data().kind()
}
/// The node's length.
pub fn len(&self) -> usize {
self.data().len()
}
/// The node's span.
pub fn span(&self) -> Span {
self.data().span()
}
/// The number of descendants, including the node itself.
pub fn descendants(&self) -> usize {
self.descendants
}
/// The node's children.
pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> {
self.children.iter()
}
/// Set a synthetic span for the node and all its descendants.
pub fn synthesize(&mut self, span: Span) {
self.data.synthesize(span);
for child in &mut self.children {
child.synthesize(span);
}
}
/// Assign span numbers `within` an interval to this node's subtree or just
/// a `range` of its children.
pub fn numberize(
&mut self,
id: SourceId,
range: Option<Range<usize>>,
within: Range<u64>,
) -> NumberingResult {
// Determine how many nodes we will number.
let descendants = match &range {
Some(range) if range.is_empty() => return Ok(()),
Some(range) => self.children[range.clone()]
.iter()
.map(SyntaxNode::descendants)
.sum::<usize>(),
None => self.descendants,
};
// Determine the distance between two neighbouring assigned numbers. If
// possible, we try to fit all numbers into the left half of `within`
// so that there is space for future insertions.
let space = within.end - within.start;
let mut stride = space / (2 * descendants as u64);
if stride == 0 {
stride = space / self.descendants as u64;
if stride == 0 {
return Err(Unnumberable);
}
}
// Number this node itself.
let mut start = within.start;
if range.is_none() {
let end = start + stride;
self.data.numberize(id, start .. end)?;
self.upper = within.end;
start = end;
}
// Number the children.
let len = self.children.len();
for child in &mut self.children[range.unwrap_or(0 .. len)] {
let end = start + child.descendants() as u64 * stride;
child.numberize(id, start .. end)?;
start = end;
}
Ok(())
}
/// The upper bound of assigned numbers in this subtree.
pub fn upper(&self) -> u64 {
self.upper
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, mut offset: usize) -> Option<Range<usize>> {
// Check whether we found it.
if let Some(range) = self.data.range(span, offset) {
return Some(range);
}
// The parent of a subtree has a smaller span number than all of its
// descendants. Therefore, we can bail out early if the target span's
// number is smaller than our number.
if span.number() < self.span().number() {
return None;
}
let mut children = self.children.iter().peekable();
while let Some(child) = children.next() {
// Every node in this child's subtree has a smaller span number than
// the next sibling. Therefore we only need to recurse if the next
// sibling's span number is larger than the target span's number.
if children
.peek()
.map_or(true, |next| next.span().number() > span.number())
{
if let Some(range) = child.range(span, offset) {
return Some(range);
}
}
offset += child.len();
}
None
}
/// The node's children, mutably.
pub(crate) fn children_mut(&mut self) -> &mut [SyntaxNode] {
&mut self.children
}
/// Replaces a range of children with a replacement.
///
/// May have mutated the children if it returns `Err(_)`.
pub(crate) fn replace_children(
&mut self,
mut range: Range<usize>,
replacement: Vec<SyntaxNode>,
) -> NumberingResult {
let superseded = &self.children[range.clone()];
// Compute the new byte length.
self.data.len = self.data.len
+ replacement.iter().map(SyntaxNode::len).sum::<usize>()
- superseded.iter().map(SyntaxNode::len).sum::<usize>();
// Compute the new number of descendants.
self.descendants = self.descendants
+ replacement.iter().map(SyntaxNode::descendants).sum::<usize>()
- superseded.iter().map(SyntaxNode::descendants).sum::<usize>();
// Determine whether we're still erroneous after the replacement. That's
// the case if
// - any of the new nodes is erroneous,
// - or if we were erroneous before due to a non-superseded node.
self.erroneous = replacement.iter().any(SyntaxNode::erroneous)
|| (self.erroneous
&& (self.children[.. range.start].iter().any(SyntaxNode::erroneous))
|| self.children[range.end ..].iter().any(SyntaxNode::erroneous));
// Perform the replacement.
let replacement_count = replacement.len();
self.children.splice(range.clone(), replacement);
range.end = range.start + replacement_count;
// Renumber the new children. Retries until it works, taking
// exponentially more children into account.
let mut left = 0;
let mut right = 0;
let max_left = range.start;
let max_right = self.children.len() - range.end;
loop {
let renumber = range.start - left .. range.end + right;
// The minimum assignable number is either
// - the upper bound of the node right before the to-be-renumbered
// children,
// - or this inner node's span number plus one if renumbering starts
// at the first child.
let start_number = renumber
.start
.checked_sub(1)
.and_then(|i| self.children.get(i))
.map_or(self.span().number() + 1, |child| child.upper());
// The upper bound for renumbering is either
// - the span number of the first child after the to-be-renumbered
// children,
// - or this node's upper bound if renumbering ends behind the last
// child.
let end_number = self
.children
.get(renumber.end)
.map_or(self.upper(), |next| next.span().number());
// Try to renumber.
let within = start_number .. end_number;
let id = self.span().source();
if self.numberize(id, Some(renumber), within).is_ok() {
return Ok(());
}
// If it didn't even work with all children, we give up.
if left == max_left && right == max_right {
return Err(Unnumberable);
}
// Exponential expansion to both sides.
left = (left + 1).next_power_of_two().min(max_left);
right = (right + 1).next_power_of_two().min(max_right);
}
}
/// Update this node after changes were made to one of its children.
pub(crate) fn update_parent(
&mut self,
prev_len: usize,
new_len: usize,
prev_descendants: usize,
new_descendants: usize,
) {
self.data.len = self.data.len + new_len - prev_len;
self.descendants = self.descendants + new_descendants - prev_descendants;
self.erroneous = self.children.iter().any(SyntaxNode::erroneous);
}
}
impl From<InnerNode> for SyntaxNode {
fn from(node: InnerNode) -> Self {
Arc::new(node).into()
}
}
impl From<Arc<InnerNode>> for SyntaxNode {
fn from(node: Arc<InnerNode>) -> Self {
Self::Inner(node)
}
}
impl Debug for InnerNode {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
self.data.fmt(f)?;
if !self.children.is_empty() {
f.write_str(" ")?;
f.debug_list().entries(&self.children).finish()?;
}
Ok(())
}
}
impl PartialEq for InnerNode {
fn eq(&self, other: &Self) -> bool {
self.data == other.data
&& self.descendants == other.descendants
&& self.erroneous == other.erroneous
&& self.children == other.children
}
}
/// Data shared between inner and leaf nodes.
#[derive(Clone, Hash)]
pub struct NodeData {
/// What kind of node this is (each kind would have its own struct in a
/// strongly typed AST).
kind: NodeKind,
/// The byte length of the node in the source.
len: usize,
/// The node's span.
span: Span,
}
impl NodeData {
/// Create new node metadata.
pub fn new(kind: NodeKind, len: usize) -> Self {
Self { len, kind, span: Span::detached() }
}
/// The node's type.
pub fn kind(&self) -> &NodeKind {
&self.kind
}
/// The node's length.
pub fn len(&self) -> usize {
self.len
}
/// The node's span.
pub fn span(&self) -> Span {
self.span
}
/// Set a synthetic span for the node.
pub fn synthesize(&mut self, span: Span) {
self.span = span;
}
/// Assign a span to the node.
pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult {
if within.start < within.end {
self.span = Span::new(id, (within.start + within.end) / 2);
Ok(())
} else {
Err(Unnumberable)
}
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> {
(self.span == span).then(|| offset .. offset + self.len())
}
}
impl From<NodeData> for SyntaxNode {
fn from(token: NodeData) -> Self {
Self::Leaf(token)
}
}
impl Debug for NodeData {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{:?}: {}", self.kind, self.len)
}
}
impl PartialEq for NodeData {
fn eq(&self, other: &Self) -> bool {
self.kind == other.kind && self.len == other.len
}
}

548
src/syntax/node.rs Normal file
View File

@ -0,0 +1,548 @@
use std::fmt::{self, Debug, Formatter};
use std::ops::Range;
use std::sync::Arc;
use super::ast::TypedNode;
use super::{NodeKind, NumberingResult, SourceId, Span, Unnumberable};
use crate::diag::SourceError;
/// An inner or leaf node in the untyped syntax tree.
#[derive(Clone, PartialEq, Hash)]
pub enum SyntaxNode {
/// A reference-counted inner node.
Inner(Arc<InnerNode>),
/// A leaf token.
Leaf(NodeData),
}
impl SyntaxNode {
/// The metadata of the node.
pub fn data(&self) -> &NodeData {
match self {
Self::Inner(inner) => &inner.data,
Self::Leaf(leaf) => leaf,
}
}
/// The type of the node.
pub fn kind(&self) -> &NodeKind {
self.data().kind()
}
/// The length of the node.
pub fn len(&self) -> usize {
self.data().len()
}
/// The number of descendants, including the node itself.
pub fn descendants(&self) -> usize {
match self {
Self::Inner(inner) => inner.descendants(),
Self::Leaf(_) => 1,
}
}
/// The span of the node.
pub fn span(&self) -> Span {
self.data().span()
}
/// Whether the node or its children contain an error.
pub fn erroneous(&self) -> bool {
match self {
Self::Inner(node) => node.erroneous,
Self::Leaf(data) => data.kind.is_error(),
}
}
/// The error messages for this node and its descendants.
pub fn errors(&self) -> Vec<SourceError> {
if !self.erroneous() {
return vec![];
}
match self.kind() {
NodeKind::Error(pos, message) => {
vec![SourceError::new(self.span(), message.clone()).with_pos(*pos)]
}
_ => self
.children()
.filter(|node| node.erroneous())
.flat_map(|node| node.errors())
.collect(),
}
}
/// The node's children.
pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> {
match self {
Self::Inner(inner) => inner.children(),
Self::Leaf(_) => [].iter(),
}
}
/// Convert the node to a typed AST node.
pub fn cast<T>(&self) -> Option<T>
where
T: TypedNode,
{
T::from_untyped(self)
}
/// Get the first child that can cast to the AST type `T`.
pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> {
self.children().find_map(Self::cast)
}
/// Get the last child that can cast to the AST type `T`.
pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> {
self.children().rev().find_map(Self::cast)
}
/// Change the type of the node.
pub fn convert(&mut self, kind: NodeKind) {
match self {
Self::Inner(inner) => {
let node = Arc::make_mut(inner);
node.erroneous |= kind.is_error();
node.data.kind = kind;
}
Self::Leaf(leaf) => leaf.kind = kind,
}
}
/// Set a synthetic span for the node and all its descendants.
pub fn synthesize(&mut self, span: Span) {
match self {
Self::Inner(inner) => Arc::make_mut(inner).synthesize(span),
Self::Leaf(leaf) => leaf.synthesize(span),
}
}
/// Assign spans to each node.
pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult {
match self {
Self::Inner(inner) => Arc::make_mut(inner).numberize(id, None, within),
Self::Leaf(leaf) => leaf.numberize(id, within),
}
}
/// The upper bound of assigned numbers in this subtree.
pub fn upper(&self) -> u64 {
match self {
Self::Inner(inner) => inner.upper(),
Self::Leaf(leaf) => leaf.span().number() + 1,
}
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> {
match self {
Self::Inner(inner) => inner.range(span, offset),
Self::Leaf(leaf) => leaf.range(span, offset),
}
}
/// Returns all leaf descendants of this node (may include itself).
///
/// This method is slow and only intended for testing.
pub fn leafs(&self) -> Vec<Self> {
if match self {
Self::Inner(inner) => inner.children.is_empty(),
Self::Leaf(_) => true,
} {
vec![self.clone()]
} else {
self.children().flat_map(Self::leafs).collect()
}
}
}
impl Default for SyntaxNode {
fn default() -> Self {
Self::Leaf(NodeData::new(NodeKind::None, 0))
}
}
impl Debug for SyntaxNode {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Self::Inner(node) => node.fmt(f),
Self::Leaf(token) => token.fmt(f),
}
}
}
/// An inner node in the untyped syntax tree.
#[derive(Clone, Hash)]
pub struct InnerNode {
/// Node metadata.
data: NodeData,
/// The number of nodes in the whole subtree, including this node.
descendants: usize,
/// Whether this node or any of its children are erroneous.
erroneous: bool,
/// The upper bound of this node's numbering range.
upper: u64,
/// This node's children, losslessly make up this node.
children: Vec<SyntaxNode>,
}
impl InnerNode {
/// Creates a new node with the given kind and a single child.
pub fn with_child(kind: NodeKind, child: impl Into<SyntaxNode>) -> Self {
Self::with_children(kind, vec![child.into()])
}
/// Creates a new node with the given kind and children.
pub fn with_children(kind: NodeKind, children: Vec<SyntaxNode>) -> Self {
let mut len = 0;
let mut descendants = 1;
let mut erroneous = kind.is_error();
for child in &children {
len += child.len();
descendants += child.descendants();
erroneous |= child.erroneous();
}
Self {
data: NodeData::new(kind, len),
descendants,
erroneous,
upper: 0,
children,
}
}
/// The node's metadata.
pub fn data(&self) -> &NodeData {
&self.data
}
/// The node's type.
pub fn kind(&self) -> &NodeKind {
self.data().kind()
}
/// The node's length.
pub fn len(&self) -> usize {
self.data().len()
}
/// The node's span.
pub fn span(&self) -> Span {
self.data().span()
}
/// The number of descendants, including the node itself.
pub fn descendants(&self) -> usize {
self.descendants
}
/// The node's children.
pub fn children(&self) -> std::slice::Iter<'_, SyntaxNode> {
self.children.iter()
}
/// Set a synthetic span for the node and all its descendants.
pub fn synthesize(&mut self, span: Span) {
self.data.synthesize(span);
for child in &mut self.children {
child.synthesize(span);
}
}
/// Assign span numbers `within` an interval to this node's subtree or just
/// a `range` of its children.
pub fn numberize(
&mut self,
id: SourceId,
range: Option<Range<usize>>,
within: Range<u64>,
) -> NumberingResult {
// Determine how many nodes we will number.
let descendants = match &range {
Some(range) if range.is_empty() => return Ok(()),
Some(range) => self.children[range.clone()]
.iter()
.map(SyntaxNode::descendants)
.sum::<usize>(),
None => self.descendants,
};
// Determine the distance between two neighbouring assigned numbers. If
// possible, we try to fit all numbers into the left half of `within`
// so that there is space for future insertions.
let space = within.end - within.start;
let mut stride = space / (2 * descendants as u64);
if stride == 0 {
stride = space / self.descendants as u64;
if stride == 0 {
return Err(Unnumberable);
}
}
// Number this node itself.
let mut start = within.start;
if range.is_none() {
let end = start + stride;
self.data.numberize(id, start .. end)?;
self.upper = within.end;
start = end;
}
// Number the children.
let len = self.children.len();
for child in &mut self.children[range.unwrap_or(0 .. len)] {
let end = start + child.descendants() as u64 * stride;
child.numberize(id, start .. end)?;
start = end;
}
Ok(())
}
/// The upper bound of assigned numbers in this subtree.
pub fn upper(&self) -> u64 {
self.upper
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, mut offset: usize) -> Option<Range<usize>> {
// Check whether we found it.
if let Some(range) = self.data.range(span, offset) {
return Some(range);
}
// The parent of a subtree has a smaller span number than all of its
// descendants. Therefore, we can bail out early if the target span's
// number is smaller than our number.
if span.number() < self.span().number() {
return None;
}
let mut children = self.children.iter().peekable();
while let Some(child) = children.next() {
// Every node in this child's subtree has a smaller span number than
// the next sibling. Therefore we only need to recurse if the next
// sibling's span number is larger than the target span's number.
if children
.peek()
.map_or(true, |next| next.span().number() > span.number())
{
if let Some(range) = child.range(span, offset) {
return Some(range);
}
}
offset += child.len();
}
None
}
/// The node's children, mutably.
pub(crate) fn children_mut(&mut self) -> &mut [SyntaxNode] {
&mut self.children
}
/// Replaces a range of children with a replacement.
///
/// May have mutated the children if it returns `Err(_)`.
pub(crate) fn replace_children(
&mut self,
mut range: Range<usize>,
replacement: Vec<SyntaxNode>,
) -> NumberingResult {
let superseded = &self.children[range.clone()];
// Compute the new byte length.
self.data.len = self.data.len
+ replacement.iter().map(SyntaxNode::len).sum::<usize>()
- superseded.iter().map(SyntaxNode::len).sum::<usize>();
// Compute the new number of descendants.
self.descendants = self.descendants
+ replacement.iter().map(SyntaxNode::descendants).sum::<usize>()
- superseded.iter().map(SyntaxNode::descendants).sum::<usize>();
// Determine whether we're still erroneous after the replacement. That's
// the case if
// - any of the new nodes is erroneous,
// - or if we were erroneous before due to a non-superseded node.
self.erroneous = replacement.iter().any(SyntaxNode::erroneous)
|| (self.erroneous
&& (self.children[.. range.start].iter().any(SyntaxNode::erroneous))
|| self.children[range.end ..].iter().any(SyntaxNode::erroneous));
// Perform the replacement.
let replacement_count = replacement.len();
self.children.splice(range.clone(), replacement);
range.end = range.start + replacement_count;
// Renumber the new children. Retries until it works, taking
// exponentially more children into account.
let mut left = 0;
let mut right = 0;
let max_left = range.start;
let max_right = self.children.len() - range.end;
loop {
let renumber = range.start - left .. range.end + right;
// The minimum assignable number is either
// - the upper bound of the node right before the to-be-renumbered
// children,
// - or this inner node's span number plus one if renumbering starts
// at the first child.
let start_number = renumber
.start
.checked_sub(1)
.and_then(|i| self.children.get(i))
.map_or(self.span().number() + 1, |child| child.upper());
// The upper bound for renumbering is either
// - the span number of the first child after the to-be-renumbered
// children,
// - or this node's upper bound if renumbering ends behind the last
// child.
let end_number = self
.children
.get(renumber.end)
.map_or(self.upper(), |next| next.span().number());
// Try to renumber.
let within = start_number .. end_number;
let id = self.span().source();
if self.numberize(id, Some(renumber), within).is_ok() {
return Ok(());
}
// If it didn't even work with all children, we give up.
if left == max_left && right == max_right {
return Err(Unnumberable);
}
// Exponential expansion to both sides.
left = (left + 1).next_power_of_two().min(max_left);
right = (right + 1).next_power_of_two().min(max_right);
}
}
/// Update this node after changes were made to one of its children.
pub(crate) fn update_parent(
&mut self,
prev_len: usize,
new_len: usize,
prev_descendants: usize,
new_descendants: usize,
) {
self.data.len = self.data.len + new_len - prev_len;
self.descendants = self.descendants + new_descendants - prev_descendants;
self.erroneous = self.children.iter().any(SyntaxNode::erroneous);
}
}
impl From<InnerNode> for SyntaxNode {
fn from(node: InnerNode) -> Self {
Arc::new(node).into()
}
}
impl From<Arc<InnerNode>> for SyntaxNode {
fn from(node: Arc<InnerNode>) -> Self {
Self::Inner(node)
}
}
impl Debug for InnerNode {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
self.data.fmt(f)?;
if !self.children.is_empty() {
f.write_str(" ")?;
f.debug_list().entries(&self.children).finish()?;
}
Ok(())
}
}
impl PartialEq for InnerNode {
fn eq(&self, other: &Self) -> bool {
self.data == other.data
&& self.descendants == other.descendants
&& self.erroneous == other.erroneous
&& self.children == other.children
}
}
/// Data shared between inner and leaf nodes.
#[derive(Clone, Hash)]
pub struct NodeData {
/// What kind of node this is (each kind would have its own struct in a
/// strongly typed AST).
pub(super) kind: NodeKind,
/// The byte length of the node in the source.
len: usize,
/// The node's span.
span: Span,
}
impl NodeData {
/// Create new node metadata.
pub fn new(kind: NodeKind, len: usize) -> Self {
Self { len, kind, span: Span::detached() }
}
/// The node's type.
pub fn kind(&self) -> &NodeKind {
&self.kind
}
/// The node's length.
pub fn len(&self) -> usize {
self.len
}
/// The node's span.
pub fn span(&self) -> Span {
self.span
}
/// Set a synthetic span for the node.
pub fn synthesize(&mut self, span: Span) {
self.span = span;
}
/// Assign a span to the node.
pub fn numberize(&mut self, id: SourceId, within: Range<u64>) -> NumberingResult {
if within.start < within.end {
self.span = Span::new(id, (within.start + within.end) / 2);
Ok(())
} else {
Err(Unnumberable)
}
}
/// If the span points into this node, convert it to a byte range.
pub fn range(&self, span: Span, offset: usize) -> Option<Range<usize>> {
(self.span == span).then(|| offset .. offset + self.len())
}
}
impl From<NodeData> for SyntaxNode {
fn from(token: NodeData) -> Self {
Self::Leaf(token)
}
}
impl Debug for NodeData {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{:?}: {}", self.kind, self.len)
}
}
impl PartialEq for NodeData {
fn eq(&self, other: &Self) -> bool {
self.kind == other.kind && self.len == other.len
}
}

3
src/parse/parser.rs → src/syntax/parser.rs
View File

@ -2,8 +2,7 @@ use std::fmt::{self, Display, Formatter};
use std::mem;
use std::ops::Range;
use super::{TokenMode, Tokens};
use crate::syntax::{ErrorPos, InnerNode, NodeData, NodeKind, SyntaxNode};
use super::{ErrorPos, InnerNode, NodeData, NodeKind, SyntaxNode, TokenMode, Tokens};
use crate::util::EcoString;
/// A convenient token-based parser.

42
src/parse/mod.rs → src/syntax/parsing.rs
View File

@ -1,18 +1,10 @@
//! Parsing and tokenization.
mod incremental;
mod parser;
mod resolve;
mod tokens;
pub use incremental::*;
pub use parser::*;
pub use tokens::*;
use std::collections::HashSet;
use crate::syntax::ast::{Assoc, BinOp, UnOp};
use crate::syntax::{ErrorPos, NodeKind, SyntaxNode};
use super::ast::{Assoc, BinOp, UnOp};
use super::{
ErrorPos, Group, Marker, NodeKind, ParseError, ParseResult, Parser, SyntaxNode,
TokenMode,
};
use crate::util::EcoString;
/// Parse a source file.
@ -32,7 +24,7 @@ pub fn parse_code(text: &str) -> SyntaxNode {
/// Reparse a code block.
///
/// Returns `Some` if all of the input was consumed.
fn reparse_code_block(
pub(crate) fn reparse_code_block(
prefix: &str,
text: &str,
end_pos: usize,
@ -56,7 +48,7 @@ fn reparse_code_block(
/// Reparse a content block.
///
/// Returns `Some` if all of the input was consumed.
fn reparse_content_block(
pub(crate) fn reparse_content_block(
prefix: &str,
text: &str,
end_pos: usize,
@ -80,7 +72,7 @@ fn reparse_content_block(
/// Reparse a sequence markup elements without the topmost node.
///
/// Returns `Some` if all of the input was consumed.
fn reparse_markup_elements(
pub(crate) fn reparse_markup_elements(
prefix: &str,
text: &str,
end_pos: usize,
@ -1146,21 +1138,3 @@ fn body(p: &mut Parser) -> ParseResult {
}
}
}
#[cfg(test)]
mod tests {
use std::fmt::Debug;
#[track_caller]
pub fn check<T>(text: &str, found: T, expected: T)
where
T: Debug + PartialEq,
{
if found != expected {
println!("source: {text:?}");
println!("expected: {expected:#?}");
println!("found: {found:#?}");
panic!("test failed");
}
}
}

3
src/parse/resolve.rs → src/syntax/resolve.rs
View File

@ -1,7 +1,6 @@
use unscanny::Scanner;
use super::{is_ident, is_newline};
use crate::syntax::RawKind;
use super::{is_ident, is_newline, RawKind};
use crate::util::EcoString;
/// Resolve all escape sequences in a string.

2
src/source.rs → src/syntax/source.rs
View File

@ -9,8 +9,8 @@ use comemo::Prehashed;
use unscanny::Scanner;
use crate::diag::SourceResult;
use crate::parse::{is_newline, parse, reparse};
use crate::syntax::ast::Markup;
use crate::syntax::{is_newline, parse, reparse};
use crate::syntax::{Span, SyntaxNode};
use crate::util::{PathExt, StrExt};

6
src/syntax/span.rs
View File

@ -2,7 +2,7 @@ use std::fmt::{self, Debug, Display, Formatter};
use std::num::NonZeroU64;
use std::ops::Range;
use crate::syntax::SourceId;
use super::SourceId;
/// A value with a span locating it in the source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
@ -42,8 +42,8 @@ impl<T: Debug> Debug for Spanned<T> {
/// A unique identifier for a syntax node.
///
/// This is used throughout the compiler to track which source section an error
/// or element stems from. Can be [mapped back](crate::source::Source::range)
/// to a byte range for user facing display.
/// or element stems from. Can be [mapped back](super::Source::range) to a byte
/// range for user facing display.
///
/// Span ids are ordered in the tree to enable quickly finding the node with
/// some id:

4
src/parse/tokens.rs → src/syntax/tokens.rs
View File

@ -4,8 +4,8 @@ use unicode_xid::UnicodeXID;
use unscanny::Scanner;
use super::resolve::{resolve_hex, resolve_raw, resolve_string};
use super::{ErrorPos, NodeKind, RawKind, Unit};
use crate::geom::{AngleUnit, LengthUnit};
use crate::syntax::{ErrorPos, NodeKind, RawKind, Unit};
use crate::util::EcoString;
/// An iterator over the tokens of a string of source code.
@ -710,8 +710,8 @@ fn is_math_id_continue(c: char) -> bool {
#[cfg(test)]
#[allow(non_snake_case)]
mod tests {
use super::super::tests::check;
use super::*;
use crate::parse::tests::check;
use ErrorPos::*;
use NodeKind::*;

6
tests/typeset.rs
View File

@ -15,15 +15,13 @@ use unscanny::Scanner;
use walkdir::WalkDir;
use typst::diag::{FileError, FileResult};
use typst::eval::{Smart, Value};
use typst::font::{Font, FontBook};
use typst::frame::{Element, Frame};
use typst::geom::{Length, RgbaColor, Sides};
use typst::library::layout::PageNode;
use typst::library::text::{TextNode, TextSize};
use typst::model::StyleMap;
use typst::source::{Source, SourceId};
use typst::syntax::SyntaxNode;
use typst::model::{Smart, StyleMap, Value};
use typst::syntax::{Source, SourceId, SyntaxNode};
use typst::util::{Buffer, PathExt};
use typst::{bail, Config, World};