konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Description lists, link syntax, and new enum syntax

Browse Source
This commit is contained in:
Laurenz 2022-09-26 15:39:32 +02:00
parent 2661f1a506
commit 704f2fbaf1
33 changed files with 905 additions and 650 deletions

35
src/eval/mod.rs

@ -195,16 +195,20 @@ impl Eval for MarkupNode {
Ok(match self {
Self::Space => Content::Space,
Self::Parbreak => Content::Parbreak,
&Self::Linebreak { justified } => Content::Linebreak { justified },
&Self::Linebreak => Content::Linebreak { justify: false },
Self::Text(text) => Content::Text(text.clone()),
&Self::Quote { double } => Content::Quote { double },
Self::Strong(strong) => strong.eval(vm)?,
Self::Emph(emph) => emph.eval(vm)?,
Self::Link(url) => {
Content::show(library::text::LinkNode::from_url(url.clone()))
}
Self::Raw(raw) => raw.eval(vm)?,
Self::Math(math) => math.eval(vm)?,
Self::Heading(heading) => heading.eval(vm)?,
Self::List(list) => list.eval(vm)?,
Self::Enum(enum_) => enum_.eval(vm)?,
Self::Desc(desc) => desc.eval(vm)?,
Self::Label(_) => Content::Empty,
Self::Ref(label) => Content::show(library::structure::RefNode(label.clone())),
Self::Expr(expr) => expr.eval(vm)?.display(),
@ -273,11 +277,8 @@ impl Eval for ListNode {
type Output = Content;
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
Ok(Content::Item(library::structure::ListItem {
kind: library::structure::UNORDERED,
number: None,
body: Box::new(self.body().eval(vm)?),
}))
let body = Box::new(self.body().eval(vm)?);
Ok(Content::Item(library::structure::ListItem::List(body)))
}
}
@ -285,11 +286,23 @@ impl Eval for EnumNode {
type Output = Content;
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
Ok(Content::Item(library::structure::ListItem {
kind: library::structure::ORDERED,
number: self.number(),
body: Box::new(self.body().eval(vm)?),
}))
let number = self.number();
let body = Box::new(self.body().eval(vm)?);
Ok(Content::Item(library::structure::ListItem::Enum(
number, body,
)))
}
}
impl Eval for DescNode {
type Output = Content;
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
let term = self.term().eval(vm)?;
let body = self.body().eval(vm)?;
Ok(Content::Item(library::structure::ListItem::Desc(Box::new(
library::structure::DescItem { term, body },
))))
}
}

1
src/library/mod.rs

@ -41,6 +41,7 @@ pub fn new() -> Scope {
std.def_node::<structure::HeadingNode>("heading");
std.def_node::<structure::ListNode>("list");
std.def_node::<structure::EnumNode>("enum");
std.def_node::<structure::DescNode>("desc");
std.def_node::<structure::TableNode>("table");
// Layout.

243
src/library/structure/list.rs

@ -1,5 +1,3 @@
use std::fmt::Write;
use unscanny::Scanner;
use crate::library::layout::{BlockSpacing, GridNode, TrackSizing};
@ -9,9 +7,7 @@ use crate::library::utility::Numbering;
/// An unordered (bulleted) or ordered (numbered) list.
#[derive(Debug, Hash)]
pub struct ListNode<const L: ListKind = UNORDERED> {
/// Where the list starts.
pub start: usize,
pub struct ListNode<const L: ListKind = LIST> {
/// If true, the items are separated by leading instead of list spacing.
pub tight: bool,
/// If true, the spacing above the list is leading instead of above spacing.
@ -20,19 +16,11 @@ pub struct ListNode<const L: ListKind = UNORDERED> {
pub items: StyleVec<ListItem>,
}
/// An item in a list.
#[derive(Clone, PartialEq, Hash)]
pub struct ListItem {
/// The kind of item.
pub kind: ListKind,
/// The number of the item.
pub number: Option<usize>,
/// The node that produces the item's body.
pub body: Box<Content>,
}
/// An ordered list.
pub type EnumNode = ListNode<ORDERED>;
pub type EnumNode = ListNode<ENUM>;
/// A description list.
pub type DescNode = ListNode<DESC>;
#[node(showable)]
impl<const L: ListKind> ListNode<L> {
@ -44,7 +32,11 @@ impl<const L: ListKind> ListNode<L> {
pub const INDENT: RawLength = RawLength::zero();
/// The space between the label and the body of each item.
#[property(resolve)]
pub const BODY_INDENT: RawLength = Em::new(0.5).into();
pub const BODY_INDENT: RawLength = Em::new(match L {
LIST | ENUM => 0.5,
DESC | _ => 1.0,
})
.into();
/// The spacing above the list.
#[property(resolve, shorthand(around))]
@ -57,19 +49,34 @@ impl<const L: ListKind> ListNode<L> {
pub const SPACING: BlockSpacing = Ratio::one().into();
fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
Ok(Content::show(Self {
start: args.named("start")?.unwrap_or(1),
tight: args.named("tight")?.unwrap_or(true),
attached: args.named("attached")?.unwrap_or(false),
items: args
let items = match L {
LIST => args
.all()?
.into_iter()
.map(|body| ListItem {
kind: L,
number: None,
body: Box::new(body),
})
.map(|body| ListItem::List(Box::new(body)))
.collect(),
ENUM => {
let mut number: usize = args.named("start")?.unwrap_or(1);
args.all()?
.into_iter()
.map(|body| {
let item = ListItem::Enum(Some(number), Box::new(body));
number += 1;
item
})
.collect()
}
DESC | _ => args
.all()?
.into_iter()
.map(|item| ListItem::Desc(Box::new(item)))
.collect(),
};
Ok(Content::show(Self {
tight: args.named("tight")?.unwrap_or(true),
attached: args.named("attached")?.unwrap_or(false),
items,
}))
}
}
@ -77,10 +84,7 @@ impl<const L: ListKind> ListNode<L> {
impl<const L: ListKind> Show for ListNode<L> {
fn unguard(&self, sel: Selector) -> ShowNode {
Self {
items: self.items.map(|item| ListItem {
body: Box::new(item.body.unguard(sel).role(Role::ListItemBody)),
..*item
}),
items: self.items.map(|item| item.unguard(sel)),
..*self
}
.pack()
@ -88,13 +92,12 @@ impl<const L: ListKind> Show for ListNode<L> {
fn encode(&self, _: StyleChain) -> Dict {
dict! {
"start" => Value::Int(self.start as i64),
"tight" => Value::Bool(self.tight),
"attached" => Value::Bool(self.attached),
"items" => Value::Array(
self.items
.items()
.map(|item| Value::Content(item.body.as_ref().clone()))
.map(|item| item.encode())
.collect()
),
}
@ -106,34 +109,54 @@ impl<const L: ListKind> Show for ListNode<L> {
styles: StyleChain,
) -> SourceResult<Content> {
let mut cells = vec![];
let mut number = self.start;
let mut number = 1;
let label = styles.get(Self::LABEL);
for (item, map) in self.items.iter() {
number = item.number.unwrap_or(number);
cells.push(LayoutNode::default());
cells.push(
label
.resolve(world, L, number)?
.styled_with_map(map.clone())
.role(Role::ListLabel)
.pack(),
);
cells.push(LayoutNode::default());
cells.push((*item.body).clone().styled_with_map(map.clone()).pack());
number += 1;
}
let indent = styles.get(Self::INDENT);
let body_indent = styles.get(Self::BODY_INDENT);
let gutter = if self.tight {
styles.get(ParNode::LEADING)
} else {
styles.get(Self::SPACING)
};
let indent = styles.get(Self::INDENT);
let body_indent = styles.get(Self::BODY_INDENT);
for (item, map) in self.items.iter() {
if let &ListItem::Enum(Some(n), _) = item {
number = n;
}
cells.push(LayoutNode::default());
let label = if L == LIST || L == ENUM {
label
.resolve(world, L, number)?
.styled_with_map(map.clone())
.role(Role::ListLabel)
.pack()
} else {
LayoutNode::default()
};
cells.push(label);
cells.push(LayoutNode::default());
let body = match &item {
ListItem::List(body) => body.as_ref().clone(),
ListItem::Enum(_, body) => body.as_ref().clone(),
ListItem::Desc(item) => Content::sequence(vec![
Content::Horizontal {
amount: (-body_indent).into(),
weak: false,
},
(item.term.clone() + Content::Text(':'.into())).strong(),
Content::Space,
item.body.clone(),
]),
};
cells.push(body.styled_with_map(map.clone()).pack());
number += 1;
}
Ok(Content::block(GridNode {
tracks: Spec::with_x(vec![
@ -165,35 +188,110 @@ impl<const L: ListKind> Show for ListNode<L> {
}
}
Ok(realized
.role(Role::List { ordered: L == ORDERED })
.spaced(above, below))
Ok(realized.role(Role::List { ordered: L == ENUM }).spaced(above, below))
}
}
/// An item in a list.
#[derive(Clone, PartialEq, Hash)]
pub enum ListItem {
/// An item of an unordered list.
List(Box<Content>),
/// An item of an ordered list.
Enum(Option<usize>, Box<Content>),
/// An item of a description list.
Desc(Box<DescItem>),
}
impl ListItem {
/// What kind of item this is.
pub fn kind(&self) -> ListKind {
match self {
Self::List(_) => LIST,
Self::Enum { .. } => ENUM,
Self::Desc { .. } => DESC,
}
}
fn unguard(&self, sel: Selector) -> Self {
match self {
Self::List(body) => Self::List(Box::new(body.unguard(sel))),
Self::Enum(number, body) => Self::Enum(*number, Box::new(body.unguard(sel))),
Self::Desc(item) => Self::Desc(Box::new(DescItem {
term: item.term.unguard(sel),
body: item.body.unguard(sel),
})),
}
}
/// Encode the item into a value.
fn encode(&self) -> Value {
match self {
Self::List(body) => Value::Content(body.as_ref().clone()),
Self::Enum(number, body) => Value::Dict(dict! {
"number" => match *number {
Some(n) => Value::Int(n as i64),
None => Value::None,
},
"body" => Value::Content(body.as_ref().clone()),
}),
Self::Desc(item) => Value::Dict(dict! {
"term" => Value::Content(item.term.clone()),
"body" => Value::Content(item.body.clone()),
}),
}
}
}
impl Debug for ListItem {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
if self.kind == UNORDERED {
f.write_char('-')?;
} else {
if let Some(number) = self.number {
write!(f, "{}", number)?;
}
f.write_char('.')?;
match self {
Self::List(body) => write!(f, "- {body:?}"),
Self::Enum(number, body) => match number {
Some(n) => write!(f, "{n}. {body:?}"),
None => write!(f, "+ {body:?}"),
},
Self::Desc(item) => item.fmt(f),
}
f.write_char(' ')?;
self.body.fmt(f)
}
}
/// A description list item.
#[derive(Clone, PartialEq, Hash)]
pub struct DescItem {
/// The term described by the list item.
pub term: Content,
/// The description of the term.
pub body: Content,
}
castable! {
DescItem,
Expected: "dictionary with `term` and `body` keys",
Value::Dict(dict) => {
let term: Content = dict.get("term")?.clone().cast()?;
let body: Content = dict.get("body")?.clone().cast()?;
Self { term, body }
},
}
impl Debug for DescItem {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "/ {:?}: {:?}", self.term, self.body)
}
}
/// How to label a list.
pub type ListKind = usize;
/// Unordered list labelling style.
pub const UNORDERED: ListKind = 0;
/// An unordered list.
pub const LIST: ListKind = 0;
/// Ordered list labelling style.
pub const ORDERED: ListKind = 1;
/// An ordered list.
pub const ENUM: ListKind = 1;
/// A description list.
pub const DESC: ListKind = 2;
/// How to label a list or enumeration.
#[derive(Debug, Clone, PartialEq, Hash)]
@ -218,8 +316,9 @@ impl Label {
) -> SourceResult<Content> {
Ok(match self {
Self::Default => match kind {
UNORDERED => Content::Text('•'.into()),
ORDERED | _ => Content::Text(format_eco!("{}.", number)),
LIST => Content::Text('•'.into()),
ENUM => Content::Text(format_eco!("{}.", number)),
DESC | _ => panic!("description lists don't have a label"),
},
Self::Pattern(prefix, numbering, upper, suffix) => {
let fmt = numbering.apply(number);

7
src/library/text/link.rs

@ -10,6 +10,13 @@ pub struct LinkNode {
pub body: Option<Content>,
}
impl LinkNode {
/// Create a link node from a URL with its bare text.
pub fn from_url(url: EcoString) -> Self {
Self { dest: Destination::Url(url), body: None }
}
}
#[node(showable)]
impl LinkNode {
/// The fill color of text in the link. Just the surrounding text color

4
src/library/text/par.rs

@ -181,8 +181,8 @@ pub struct LinebreakNode;
#[node]
impl LinebreakNode {
fn construct(_: &mut Vm, args: &mut Args) -> SourceResult<Content> {
let justified = args.named("justified")?.unwrap_or(false);
Ok(Content::Linebreak { justified })
let justify = args.named("justify")?.unwrap_or(false);
Ok(Content::Linebreak { justify })
}
}

2
src/library/text/raw.rs

@ -91,7 +91,7 @@ impl Show for RawNode {
let mut highlighter = HighlightLines::new(syntax, &THEME);
for (i, line) in self.text.lines().enumerate() {
if i != 0 {
seq.push(Content::Linebreak { justified: false });
seq.push(Content::Linebreak { justify: false });
}
for (style, piece) in

25
src/model/content.rs

@ -14,7 +14,7 @@ use super::{
use crate::diag::StrResult;
use crate::library::layout::{FlowChild, FlowNode, PageNode, PlaceNode, Spacing};
use crate::library::prelude::*;
use crate::library::structure::{DocNode, ListItem, ListNode, ORDERED, UNORDERED};
use crate::library::structure::{DocNode, ListItem, ListNode, DESC, ENUM, LIST};
use crate::library::text::{
DecoNode, EmphNode, ParChild, ParNode, StrongNode, UNDERLINE,
};
@ -62,7 +62,7 @@ pub enum Content {
/// A word space.
Space,
/// A forced line break.
Linebreak { justified: bool },
Linebreak { justify: bool },
/// Horizontal spacing.
Horizontal { amount: Spacing, weak: bool },
/// Plain text.
@ -264,7 +264,7 @@ impl Debug for Content {
match self {
Self::Empty => f.pad("Empty"),
Self::Space => f.pad("Space"),
Self::Linebreak { justified } => write!(f, "Linebreak({justified})"),
Self::Linebreak { justify } => write!(f, "Linebreak({justify})"),
Self::Horizontal { amount, weak } => {
write!(f, "Horizontal({amount:?}, {weak})")
}
@ -633,8 +633,8 @@ impl<'a> ParBuilder<'a> {
Content::Space => {
self.0.weak(ParChild::Text(' '.into()), styles, 2);
}
&Content::Linebreak { justified } => {
let c = if justified { '\u{2028}' } else { '\n' };
&Content::Linebreak { justify } => {
let c = if justify { '\u{2028}' } else { '\n' };
self.0.destructive(ParChild::Text(c.into()), styles);
}
&Content::Horizontal { amount, weak } => {
@ -734,7 +734,7 @@ impl<'a> ListBuilder<'a> {
.items
.items()
.next()
.map_or(true, |first| item.kind == first.kind) =>
.map_or(true, |first| item.kind() == first.kind()) =>
{
self.items.push(item.clone(), styles);
self.tight &= self.staged.drain(..).all(|(t, _)| *t != Content::Parbreak);
@ -751,21 +751,16 @@ impl<'a> ListBuilder<'a> {
fn finish(self, parent: &mut Builder<'a>) -> SourceResult<()> {
let (items, shared) = self.items.finish();
let kind = match items.items().next() {
Some(item) => item.kind,
Some(item) => item.kind(),
None => return Ok(()),
};
let start = 1;
let tight = self.tight;
let attached = tight && self.attachable;
let content = match kind {
UNORDERED => {
Content::show(ListNode::<UNORDERED> { start, tight, attached, items })
}
ORDERED | _ => {
Content::show(ListNode::<ORDERED> { start, tight, attached, items })
}
LIST => Content::show(ListNode::<LIST> { tight, attached, items }),
ENUM => Content::show(ListNode::<ENUM> { tight, attached, items }),
DESC | _ => Content::show(ListNode::<DESC> { tight, attached, items }),
};
let stored = parent.scratch.templates.alloc(content);

7
src/model/property.rs

@ -9,7 +9,7 @@ use super::{Interruption, NodeId, StyleChain};
use crate::eval::{RawLength, Smart};
use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec};
use crate::library::layout::PageNode;
use crate::library::structure::{EnumNode, ListNode};
use crate::library::structure::{DescNode, EnumNode, ListNode};
use crate::library::text::ParNode;
use crate::util::ReadableTypeId;
@ -68,7 +68,10 @@ impl Property {
Some(Interruption::Page)
} else if self.is_of::<ParNode>() {
Some(Interruption::Par)
} else if self.is_of::<ListNode>() || self.is_of::<EnumNode>() {
} else if self.is_of::<ListNode>()
|| self.is_of::<EnumNode>()
|| self.is_of::<DescNode>()
{
Some(Interruption::List)
} else {
None

7
src/model/recipe.rs

@ -5,7 +5,7 @@ use comemo::Tracked;
use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry};
use crate::diag::SourceResult;
use crate::eval::{Args, Func, Regex, Value};
use crate::library::structure::{EnumNode, ListNode};
use crate::library::structure::{DescNode, EnumNode, ListNode};
use crate::syntax::Spanned;
use crate::World;
@ -93,7 +93,10 @@ impl Recipe {
/// What kind of structure the property interrupts.
pub fn interruption(&self) -> Option<Interruption> {
if let Pattern::Node(id) = self.pattern {
if id == NodeId::of::<ListNode>() || id == NodeId::of::<EnumNode>() {
if id == NodeId::of::<ListNode>()
|| id == NodeId::of::<EnumNode>()
|| id == NodeId::of::<DescNode>()
{
return Some(Interruption::List);
}
}

32
src/parse/incremental.rs

@ -407,18 +407,18 @@ mod tests {
test("", 0..0, "do it", 0..5);
test("a d e", 1 .. 3, " b c d", 0 .. 9);
test("*~ *", 2..2, "*", 0..5);
test("_1_\n2a\n3", 5..5, "4", 0..7);
test("_1_\n2a\n3~", 8..8, "4", 5..10);
test("_1_\n2a\n3", 5..5, "4", 4..7);
test("_1_\n2a\n3~", 8..8, "4", 4..10);
test("_1_ 2 3a\n4", 7..7, "5", 0..9);
test("* {1+2} *", 5..6, "3", 2..7);
test("a #f() e", 1 .. 6, " b c d", 0 .. 9);
test("a\nb\nc\nd\ne\n", 5 .. 5, "c", 2 .. 7);
test("a\n\nb\n\nc\n\nd\n\ne\n", 7 .. 7, "c", 3 .. 10);
test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 6..20);
test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 4..20);
test("~~ {a} ~~", 4 .. 5, "b", 3 .. 6);
test("{(0, 1, 2)}", 5 .. 6, "11pt", 0..14);
test("\n= A heading", 4 .. 4, "n evocative", 0 .. 23);
test("for~your~thing", 9 .. 9, "a", 4 .. 15);
test("for~your~thing", 9 .. 9, "a", 0 .. 15);
test("a your thing a", 6 .. 7, "a", 0 .. 14);
test("{call(); abc}", 7 .. 7, "[]", 0 .. 15);
test("#call() abc", 7 .. 7, "[]", 0 .. 10);
@ -429,17 +429,17 @@ mod tests {
test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 33 .. 40);
test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 0 .. 33);
test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 0 .. 33);
test("hello~~{x}", 7 .. 10, "#f()", 5 .. 11);
test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 5 .. 25);
test("understanding `code` is complicated", 15 .. 15, "C ", 14 .. 22);
test("hello~~{x}", 7 .. 10, "#f()", 0 .. 11);
test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 0 .. 25);
test("understanding `code` is complicated", 15 .. 15, "C ", 0 .. 22);
test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17);
test(r#"a ```typst hello``` b"#, 16 .. 17, "", 2 .. 18);
test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18);
test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 18);
test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18);
test("#for", 4 .. 4, "//", 0 .. 6);
test("#show a: f as b..", 16..16, "c", 0..18);
test("a\n#let \nb", 7 .. 7, "i", 2 .. 9);
test("a\n#for i \nb", 9 .. 9, "in", 2 .. 12);
test("a~https://fun/html", 13..14, "n", 2..18);
test("a~https://fun/html", 13..14, "n", 0..18);
}
#[test]
@ -452,7 +452,7 @@ mod tests {
test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19);
test("#let x = (1, 2 + ;~ Five\r\n\r", 20 .. 23, "2.", 0 .. 23);
test("hey #myfriend", 4 .. 4, "\\", 0 .. 14);
test("hey #myfriend", 4 .. 4, "\\", 3 .. 6);
test("hey #myfriend", 4 .. 4, "\\", 0 .. 6);
test("= foo\nbar\n - a\n - b", 6 .. 9, "", 0 .. 11);
test("= foo\n bar\n baz", 6 .. 8, "", 0 .. 9);
test(" // hi", 1 .. 1, " ", 0 .. 7);
@ -461,12 +461,12 @@ mod tests {
#[test]
fn test_parse_incremental_type_invariants() {
test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22);
test("a #let x = 1 {5}", 3 .. 6, "if", 2 .. 11);
test("a #for x in array {x}", 18 .. 21, "[#x]", 0 .. 22);
test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 11);
test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16);
test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 13);
test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18);
test("a // b c #f()", 3 .. 4, "", 2 .. 12);
test("a // b c #f()", 3 .. 4, "", 0 .. 12);
test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12);
test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13);
test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26);
@ -477,8 +477,8 @@ mod tests {
#[test]
fn test_parse_incremental_wrongly_or_unclosed_things() {
test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6);
test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12);
test(r"this \u{abcd} that", 12 .. 13, "", 5 .. 17);
test(r"this \u{abcd}", 8 .. 9, "", 0 .. 12);
test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17);
test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24);
test("a b c", 1 .. 1, " /* letters */", 0 .. 19);
test("a b c", 1 .. 1, " /* letters", 0 .. 16);

69
src/parse/mod.rs

@ -162,11 +162,6 @@ fn markup(p: &mut Parser, mut at_start: bool) {
});
}
/// Parse a single line of markup.
fn markup_line(p: &mut Parser) {
markup_indented(p, usize::MAX);
}
/// Parse markup that stays right of the given `column`.
fn markup_indented(p: &mut Parser, min_indent: usize) {
p.eat_while(|t| match t {
@ -185,7 +180,6 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
{
break;
}
Some(NodeKind::Label(_)) => break,
_ => {}
}
@ -195,6 +189,33 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
marker.end(p, NodeKind::Markup { min_indent });
}
/// Parse a line of markup that can prematurely end if `f` returns true.
fn markup_line<F>(p: &mut Parser, mut f: F)
where
F: FnMut(&NodeKind) -> bool,
{
p.eat_while(|t| match t {
NodeKind::Space { newlines } => *newlines == 0,
NodeKind::LineComment | NodeKind::BlockComment => true,
_ => false,
});
p.perform(NodeKind::Markup { min_indent: usize::MAX }, |p| {
let mut at_start = false;
while let Some(kind) = p.peek() {
if let NodeKind::Space { newlines: (1 ..) } = kind {
break;
}
if f(kind) {
break;
}
markup_node(p, &mut at_start);
}
});
}
/// Parse a markup node.
fn markup_node(p: &mut Parser, at_start: &mut bool) {
let token = match p.peek() {
@ -226,6 +247,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
| NodeKind::Ellipsis
| NodeKind::Quote { .. }
| NodeKind::Escape(_)
| NodeKind::Link(_)
| NodeKind::Raw(_)
| NodeKind::Math(_)
| NodeKind::Label(_)
@ -233,12 +255,22 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
p.eat();
}
// Grouping markup.
// Strong, emph, heading.
NodeKind::Star => strong(p),
NodeKind::Underscore => emph(p),
NodeKind::Eq => heading(p, *at_start),
// Lists.
NodeKind::Minus => list_node(p, *at_start),
NodeKind::EnumNumbering(_) => enum_node(p, *at_start),
NodeKind::Plus | NodeKind::EnumNumbering(_) => enum_node(p, *at_start),
NodeKind::Slash => {
desc_node(p, *at_start).ok();
}
NodeKind::Colon => {
let marker = p.marker();
p.eat();
marker.convert(p, NodeKind::Text(':'.into()));
}
// Hashtag + keyword / identifier.
NodeKind::Ident(_)
@ -293,7 +325,7 @@ fn heading(p: &mut Parser, at_start: bool) {
if at_start && p.peek().map_or(true, |kind| kind.is_space()) {
p.eat_while(|kind| *kind == NodeKind::Space { newlines: 0 });
markup_line(p);
markup_line(p, |kind| matches!(kind, NodeKind::Label(_)));
marker.end(p, NodeKind::Heading);
} else {
let text = p.get(current_start .. p.prev_end()).into();
@ -331,6 +363,25 @@ fn enum_node(p: &mut Parser, at_start: bool) {
}
}
/// Parse a single description list item.
fn desc_node(p: &mut Parser, at_start: bool) -> ParseResult {
let marker = p.marker();
let text: EcoString = p.peek_src().into();
p.eat();
let min_indent = p.column(p.prev_end());
if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() {
markup_line(p, |node| matches!(node, NodeKind::Colon));
p.expect(NodeKind::Colon)?;
markup_indented(p, min_indent);
marker.end(p, NodeKind::Desc);
} else {
marker.convert(p, NodeKind::Text(text));
}
Ok(())
}
/// Parse an expression within a markup mode.
fn markup_expr(p: &mut Parser) {
// Does the expression need termination or can content follow directly?

2
src/parse/resolve.rs

@ -26,14 +26,12 @@ pub fn resolve_string(string: &str) -> EcoString {
// TODO: Error if closing brace is missing.
let sequence = s.eat_while(char::is_ascii_hexdigit);
let _terminated = s.eat_if('}');
match resolve_hex(sequence) {
Some(c) => out.push(c),
None => out.push_str(s.from(start)),
}
}
// TODO: Error for invalid escape sequence.
_ => out.push_str(s.from(start)),
}
}

519
src/parse/tokens.rs

@ -103,6 +103,11 @@ impl<'s> Iterator for Tokens<'s> {
let start = self.s.cursor();
let c = self.s.eat()?;
Some(match c {
// Comments.
'/' if self.s.eat_if('/') => self.line_comment(),
'/' if self.s.eat_if('*') => self.block_comment(),
'*' if self.s.eat_if('/') => NodeKind::Unknown("*/".into()),
// Blocks.
'{' => NodeKind::LeftBrace,
'}' => NodeKind::RightBrace,
@ -110,15 +115,7 @@ impl<'s> Iterator for Tokens<'s> {
']' => NodeKind::RightBracket,
// Whitespace.
' ' if self.s.done() || !self.s.at(char::is_whitespace) => {
NodeKind::Space { newlines: 0 }
}
c if c.is_whitespace() => self.whitespace(),
// Comments with special case for URLs.
'/' if self.s.eat_if('*') => self.block_comment(),
'/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(),
'*' if self.s.eat_if('/') => NodeKind::Unknown(self.s.from(start).into()),
c if c.is_whitespace() => self.whitespace(c),
// Other things.
_ => match self.mode {
@ -130,122 +127,50 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
#[inline]
fn markup(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Escape sequences.
'\\' => self.backslash(),
// Keywords and identifiers.
'#' => self.hash(),
// Markup.
'~' => NodeKind::NonBreakingSpace,
'-' => self.hyph(),
'.' if self.s.eat_if("..") => NodeKind::Ellipsis,
'\'' => NodeKind::Quote { double: false },
'"' => NodeKind::Quote { double: true },
'*' if !self.in_word() => NodeKind::Star,
'_' if !self.in_word() => NodeKind::Underscore,
'`' => self.raw(),
'=' => NodeKind::Eq,
'$' => self.math(),
'<' => self.label(),
'@' => self.reference(),
c if c == '.' || c.is_ascii_digit() => self.numbering(start, c),
// Plain text.
_ => self.text(start),
fn line_comment(&mut self) -> NodeKind {
self.s.eat_until(is_newline);
if self.s.peek().is_none() {
self.terminated = false;
}
NodeKind::LineComment
}
fn code(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Parens.
'(' => NodeKind::LeftParen,
')' => NodeKind::RightParen,
fn block_comment(&mut self) -> NodeKind {
let mut state = '_';
let mut depth = 1;
self.terminated = false;
// Length two.
'=' if self.s.eat_if('=') => NodeKind::EqEq,
'!' if self.s.eat_if('=') => NodeKind::ExclEq,
'<' if self.s.eat_if('=') => NodeKind::LtEq,
'>' if self.s.eat_if('=') => NodeKind::GtEq,
'+' if self.s.eat_if('=') => NodeKind::PlusEq,
'-' if self.s.eat_if('=') => NodeKind::HyphEq,
'*' if self.s.eat_if('=') => NodeKind::StarEq,
'/' if self.s.eat_if('=') => NodeKind::SlashEq,
'.' if self.s.eat_if('.') => NodeKind::Dots,
'=' if self.s.eat_if('>') => NodeKind::Arrow,
// Length one.
',' => NodeKind::Comma,
';' => NodeKind::Semicolon,
':' => NodeKind::Colon,
'+' => NodeKind::Plus,
'-' => NodeKind::Minus,
'*' => NodeKind::Star,
'/' => NodeKind::Slash,
'=' => NodeKind::Eq,
'<' => NodeKind::Lt,
'>' => NodeKind::Gt,
'.' if self.s.done() || !self.s.at(char::is_ascii_digit) => NodeKind::Dot,
// Identifiers.
c if is_id_start(c) => self.ident(start),
// Numbers.
c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => {
self.number(start, c)
// Find the first `*/` that does not correspond to a nested `/*`.
while let Some(c) = self.s.eat() {
state = match (state, c) {
('*', '/') => {
depth -= 1;
if depth == 0 {
self.terminated = true;
break;
}
'_'
}
('/', '*') => {
depth += 1;
'_'
}
('/', '/') => {
self.line_comment();
'_'
}
_ => c,
}
// Strings.
'"' => self.string(),
_ => NodeKind::Unknown(self.s.from(start).into()),
}
NodeKind::BlockComment
}
#[inline]
fn text(&mut self, start: usize) -> NodeKind {
macro_rules! table {
($($c:literal)|*) => {{
let mut t = [false; 128];
$(t[$c as usize] = true;)*
t
}}
fn whitespace(&mut self, c: char) -> NodeKind {
if c == ' ' && !self.s.at(char::is_whitespace) {
return NodeKind::Space { newlines: 0 };
}
const TABLE: [bool; 128] = table! {
// Ascii whitespace.
' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' |
// Comments, parentheses, code.
'/' | '[' | ']' | '{' | '}' | '#' |
// Markup
'~' | '-' | '.' | '\'' | '"' | '*' | '_' | '`' | '$' | '\\'
};
loop {
self.s.eat_until(|c: char| {
TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
});
// Allow a single space, optionally preceded by . or - if something
// alphanumeric follows directly. This leads to less text nodes,
// which is good for performance.
let mut s = self.s;
s.eat_if(['.', '-']);
s.eat_if(' ');
if !s.at(char::is_alphanumeric) {
break;
}
self.s = s;
}
NodeKind::Text(self.s.from(start).into())
}
fn whitespace(&mut self) -> NodeKind {
self.s.uneat();
// Count the number of newlines.
@ -267,25 +192,84 @@ impl<'s> Tokens<'s> {
NodeKind::Space { newlines }
}
fn backslash(&mut self) -> NodeKind {
let c = match self.s.peek() {
Some(c) => c,
None => return NodeKind::Linebreak { justified: false },
#[inline]
fn markup(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Escape sequences.
'\\' => self.backslash(),
// Single-char things.
'~' => NodeKind::NonBreakingSpace,
'.' if self.s.eat_if("..") => NodeKind::Ellipsis,
'\'' => NodeKind::Quote { double: false },
'"' => NodeKind::Quote { double: true },
'*' if !self.in_word() => NodeKind::Star,
'_' if !self.in_word() => NodeKind::Underscore,
'=' => NodeKind::Eq,
'+' => NodeKind::Plus,
'/' => NodeKind::Slash,
':' => NodeKind::Colon,
// Multi-char things.
'#' => self.hash(start),
'-' => self.hyph(),
'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => {
self.link(start)
}
'`' => self.raw(),
'$' => self.math(),
c if c.is_ascii_digit() => self.numbering(start),
'<' => self.label(),
'@' => self.reference(start),
// Plain text.
_ => self.text(start),
}
}
#[inline]
fn text(&mut self, start: usize) -> NodeKind {
macro_rules! table {
($(|$c:literal)*) => {{
let mut t = [false; 128];
$(t[$c as usize] = true;)*
t
}}
}
const TABLE: [bool; 128] = table! {
| ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | '\\' | '/'
| '[' | ']' | '{' | '}' | '~' | '-' | '.' | '\'' | '"'
| '*' | '_' | ':' | 'h' | '`' | '$' | '<' | '>' | '@' | '#'
};
match c {
// Backslash and comments.
'\\' | '/' |
// Parenthesis and hashtag.
'[' | ']' | '{' | '}' | '#' |
// Markup.
'~' | '-' | '.' | ':' |
'\'' | '"' | '*' | '_' | '`' | '$' | '=' |
'<' | '>' | '@' => {
self.s.expect(c);
NodeKind::Escape(c)
loop {
self.s.eat_until(|c: char| {
TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace())
});
// Continue with the same text node if the thing would become text
// anyway.
let mut s = self.s;
match s.eat() {
Some('/') if !s.at(['/', '*']) => {}
Some(' ') if s.at(char::is_alphanumeric) => {}
Some('-') if !s.at(['-', '?']) => {}
Some('.') if !s.at("..") => {}
Some('h') if !s.at("ttp://") && !s.at("ttps://") => {}
Some('@' | '#') if !s.at(is_id_start) => {}
_ => break,
}
'u' if self.s.eat_if("u{") => {
self.s = s;
}
NodeKind::Text(self.s.from(start).into())
}
fn backslash(&mut self) -> NodeKind {
match self.s.peek() {
Some('u') if self.s.eat_if("u{") => {
let sequence = self.s.eat_while(char::is_ascii_alphanumeric);
if self.s.eat_if('}') {
if let Some(c) = resolve_hex(sequence) {
@ -298,26 +282,23 @@ impl<'s> Tokens<'s> {
}
} else {
self.terminated = false;
NodeKind::Error(
SpanPos::End,
"expected closing brace".into(),
)
NodeKind::Error(SpanPos::End, "expected closing brace".into())
}
}
// Linebreaks.
c if c.is_whitespace() => NodeKind::Linebreak { justified: false },
'+' => {
self.s.expect(c);
NodeKind::Linebreak { justified: true }
}
Some(c) if c.is_whitespace() => NodeKind::Linebreak,
None => NodeKind::Linebreak,
// Just the backslash.
_ => NodeKind::Text('\\'.into()),
// Escapes.
Some(c) => {
self.s.expect(c);
NodeKind::Escape(c)
}
}
}
fn hash(&mut self) -> NodeKind {
fn hash(&mut self, start: usize) -> NodeKind {
if self.s.at(is_id_start) {
let read = self.s.eat_while(is_id_continue);
match keyword(read) {
@ -325,7 +306,7 @@ impl<'s> Tokens<'s> {
None => NodeKind::Ident(read.into()),
}
} else {
NodeKind::Text('#'.into())
self.text(start)
}
}
@ -343,19 +324,26 @@ impl<'s> Tokens<'s> {
}
}
fn numbering(&mut self, start: usize, c: char) -> NodeKind {
let number = if c != '.' {
self.s.eat_while(char::is_ascii_digit);
let read = self.s.from(start);
if !self.s.eat_if('.') {
return NodeKind::Text(self.s.from(start).into());
}
read.parse().ok()
} else {
None
};
fn in_word(&self) -> bool {
let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
let prev = self.s.scout(-2);
let next = self.s.peek();
alphanumeric(prev) && alphanumeric(next)
}
NodeKind::EnumNumbering(number)
fn link(&mut self, start: usize) -> NodeKind {
#[rustfmt::skip]
self.s.eat_while(|c: char| matches!(c,
| '0' ..= '9'
| 'a' ..= 'z'
| 'A' ..= 'Z'
| '~' | '/' | '%' | '?' | '#' | '&' | '+' | '='
| '\'' | '.' | ',' | ';'
));
if self.s.scout(-1) == Some('.') {
self.s.uneat();
}
NodeKind::Link(self.s.from(start).into())
}
fn raw(&mut self) -> NodeKind {
@ -376,7 +364,6 @@ impl<'s> Tokens<'s> {
}
let start = self.s.cursor();
let mut found = 0;
while found < backticks {
match self.s.eat() {
@ -394,10 +381,9 @@ impl<'s> Tokens<'s> {
self.s.get(start .. end),
)))
} else {
self.terminated = false;
let remaining = backticks - found;
let noun = if remaining == 1 { "backtick" } else { "backticks" };
self.terminated = false;
NodeKind::Error(
SpanPos::End,
if found == 0 {
@ -410,53 +396,40 @@ impl<'s> Tokens<'s> {
}
fn math(&mut self) -> NodeKind {
let mut display = false;
if self.s.eat_if('[') {
display = true;
}
let start = self.s.cursor();
let mut escaped = false;
let mut dollar = !display;
let terminated = loop {
match self.s.eat() {
Some('$') if !escaped && dollar => break true,
Some(']') if !escaped => dollar = true,
Some(c) => {
dollar = !display;
escaped = c == '\\' && !escaped;
}
None => break false,
let formula = self.s.eat_until(|c| {
if c == '$' && !escaped {
true
} else {
escaped = c == '\\' && !escaped;
false
}
};
});
let end = self.s.cursor()
- match (terminated, display) {
(false, _) => 0,
(true, false) => 1,
(true, true) => 2,
};
let display = formula.len() >= 2
&& formula.starts_with(char::is_whitespace)
&& formula.ends_with(char::is_whitespace);
if terminated {
NodeKind::Math(Arc::new(MathNode {
formula: self.s.get(start .. end).into(),
display,
}))
if self.s.eat_if('$') {
NodeKind::Math(Arc::new(MathNode { formula: formula.into(), display }))
} else {
self.terminated = false;
NodeKind::Error(
SpanPos::End,
if !display || (!escaped && dollar) {
"expected closing dollar sign".into()
} else {
"expected closing bracket and dollar sign".into()
},
)
NodeKind::Error(SpanPos::End, "expected dollar sign".into())
}
}
fn numbering(&mut self, start: usize) -> NodeKind {
self.s.eat_while(char::is_ascii_digit);
let read = self.s.from(start);
if self.s.eat_if('.') {
if let Ok(number) = read.parse() {
return NodeKind::EnumNumbering(number);
}
}
self.text(start)
}
fn label(&mut self) -> NodeKind {
let label = self.s.eat_while(is_id_continue);
if self.s.eat_if('>') {
@ -471,12 +444,59 @@ impl<'s> Tokens<'s> {
}
}
fn reference(&mut self) -> NodeKind {
fn reference(&mut self, start: usize) -> NodeKind {
let label = self.s.eat_while(is_id_continue);
if !label.is_empty() {
NodeKind::Ref(label.into())
} else {
NodeKind::Error(SpanPos::Full, "label cannot be empty".into())
self.text(start)
}
}
fn code(&mut self, start: usize, c: char) -> NodeKind {
match c {
// Parentheses.
'(' => NodeKind::LeftParen,
')' => NodeKind::RightParen,
// Two-char operators.
'=' if self.s.eat_if('=') => NodeKind::EqEq,
'!' if self.s.eat_if('=') => NodeKind::ExclEq,
'<' if self.s.eat_if('=') => NodeKind::LtEq,
'>' if self.s.eat_if('=') => NodeKind::GtEq,
'+' if self.s.eat_if('=') => NodeKind::PlusEq,
'-' if self.s.eat_if('=') => NodeKind::HyphEq,
'*' if self.s.eat_if('=') => NodeKind::StarEq,
'/' if self.s.eat_if('=') => NodeKind::SlashEq,
'.' if self.s.eat_if('.') => NodeKind::Dots,
'=' if self.s.eat_if('>') => NodeKind::Arrow,
// Single-char operators.
',' => NodeKind::Comma,
';' => NodeKind::Semicolon,
':' => NodeKind::Colon,
'+' => NodeKind::Plus,
'-' => NodeKind::Minus,
'*' => NodeKind::Star,
'/' => NodeKind::Slash,
'=' => NodeKind::Eq,
'<' => NodeKind::Lt,
'>' => NodeKind::Gt,
'.' if !self.s.at(char::is_ascii_digit) => NodeKind::Dot,
// Identifiers.
c if is_id_start(c) => self.ident(start),
// Numbers.
c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => {
self.number(start, c)
}
// Strings.
'"' => self.string(),
// Invalid token.
_ => NodeKind::Unknown(self.s.from(start).into()),
}
}
@ -543,18 +563,18 @@ impl<'s> Tokens<'s> {
}
}
fn string(&mut self) -> NodeKind {
let mut escaped = false;
let string = resolve_string(self.s.eat_until(|c| {
let verbatim = self.s.eat_until(|c| {
if c == '"' && !escaped {
true
} else {
escaped = c == '\\' && !escaped;
false
}
}));
});
let string = resolve_string(verbatim);
if self.s.eat_if('"') {
NodeKind::Str(string)
} else {
@ -562,56 +582,6 @@ impl<'s> Tokens<'s> {
NodeKind::Error(SpanPos::End, "expected quote".into())
}
}
fn line_comment(&mut self) -> NodeKind {
self.s.eat_until(is_newline);
if self.s.peek().is_none() {
self.terminated = false;
}
NodeKind::LineComment
}
fn block_comment(&mut self) -> NodeKind {
let mut state = '_';
let mut depth = 1;
self.terminated = false;
// Find the first `*/` that does not correspond to a nested `/*`.
while let Some(c) = self.s.eat() {
state = match (state, c) {
('*', '/') => {
depth -= 1;
if depth == 0 {
self.terminated = true;
break;
}
'_'
}
('/', '*') => {
depth += 1;
'_'
}
('/', '/') => {
self.line_comment();
'_'
}
_ => c,
}
}
NodeKind::BlockComment
}
fn in_word(&self) -> bool {
let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
let prev = self.s.scout(-2);
let next = self.s.peek();
alphanumeric(prev) && alphanumeric(next)
}
fn maybe_in_url(&self) -> bool {
self.mode == TokenMode::Markup && self.s.before().ends_with(":/")
}
}
fn keyword(ident: &str) -> Option<NodeKind> {
@ -872,14 +842,14 @@ mod tests {
#[test]
fn test_tokenize_text() {
// Test basic text.
t!(Markup[" /"]: "hello" => Text("hello"));
t!(Markup[" /"]: "hello-world" => Text("hello-world"));
t!(Markup[" /"]: "hello" => Text("hello"));
t!(Markup[" /"]: "reha-world" => Text("reha-world"));
// Test code symbols in text.
t!(Markup[" /"]: "a():\"b" => Text("a():"), Quote { double: true }, Text("b"));
t!(Markup[" /"]: ";:,|/+" => Text(";:,|"), Text("/+"));
t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, Quote { double: true }, Text("b"));
t!(Markup[" /"]: ";,|/+" => Text(";,|/+"));
t!(Markup[" /"]: "=-a" => Eq, Minus, Text("a"));
t!(Markup[" "]: "#123" => Text("#"), Text("123"));
t!(Markup[" "]: "#123" => Text("#123"));
// Test text ends.
t!(Markup[""]: "hello " => Text("hello"), Space(0));
@ -904,11 +874,9 @@ mod tests {
t!(Markup: r"\`" => Escape('`'));
t!(Markup: r"\$" => Escape('$'));
t!(Markup: r"\#" => Escape('#'));
// Test unescapable symbols.
t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a"));
t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u"));
t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1"));
t!(Markup: r"\a" => Escape('a'));
t!(Markup: r"\u" => Escape('u'));
t!(Markup: r"\1" => Escape('1'));
// Test basic unicode escapes.
t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence"));
@ -930,16 +898,15 @@ mod tests {
t!(Markup: "_" => Underscore);
t!(Markup[""]: "===" => Eq, Eq, Eq);
t!(Markup["a1/"]: "= " => Eq, Space(0));
t!(Markup[" "]: r"\" => Linebreak { justified: false });
t!(Markup[" "]: r"\+" => Linebreak { justified: true });
t!(Markup[" "]: r"\" => Linebreak);
t!(Markup: "~" => NonBreakingSpace);
t!(Markup["a1/"]: "-?" => Shy);
t!(Markup["a "]: r"a--" => Text("a"), EnDash);
t!(Markup["a1/"]: "- " => Minus, Space(0));
t!(Markup[" "]: "." => EnumNumbering(None));
t!(Markup[" "]: "1." => EnumNumbering(Some(1)));
t!(Markup[" "]: "1.a" => EnumNumbering(Some(1)), Text("a"));
t!(Markup[" /"]: "a1." => Text("a1"), EnumNumbering(None));
t!(Markup[" "]: "+" => Plus);
t!(Markup[" "]: "1." => EnumNumbering(1));
t!(Markup[" "]: "1.a" => EnumNumbering(1), Text("a"));
t!(Markup[" /"]: "a1." => Text("a1."));
}
#[test]
@ -995,7 +962,7 @@ mod tests {
for (s, t) in list.clone() {
t!(Markup[" "]: format!("#{}", s) => t);
t!(Markup[" "]: format!("#{0}#{0}", s) => t, t);
t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s));
t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}")));
}
for (s, t) in list {
@ -1037,18 +1004,16 @@ mod tests {
t!(Markup: "$$" => Math("", false));
t!(Markup: "$x$" => Math("x", false));
t!(Markup: r"$\\$" => Math(r"\\", false));
t!(Markup: "$[x + y]$" => Math("x + y", true));
t!(Markup: r"$[\\]$" => Math(r"\\", true));
t!(Markup: r"$[\\]$" => Math(r"[\\]", false));
t!(Markup: "$ x + y $" => Math(" x + y ", true));
// Test unterminated.
t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign"));
t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign"));
t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign"));
t!(Markup[""]: "$x" => Error(End, "expected dollar sign"));
t!(Markup[""]: "$[x]\n" => Error(End, "expected dollar sign"));
// Test escape sequences.
t!(Markup: r"$\$x$" => Math(r"\$x", false));
t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true));
t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign"));
t!(Markup: r"$\$x$" => Math(r"\$x", false));
t!(Markup: r"$\ \$ $" => Math(r"\ \$ ", false));
}
#[test]

54
src/syntax/ast.rs

@ -63,9 +63,7 @@ impl Markup {
self.0.children().filter_map(|node| match node.kind() {
NodeKind::Space { newlines: (2 ..) } => Some(MarkupNode::Parbreak),
NodeKind::Space { .. } => Some(MarkupNode::Space),
&NodeKind::Linebreak { justified } => {
Some(MarkupNode::Linebreak { justified })
}
NodeKind::Linebreak => Some(MarkupNode::Linebreak),
NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())),
NodeKind::Escape(c) => Some(MarkupNode::Text((*c).into())),
NodeKind::NonBreakingSpace => Some(MarkupNode::Text('\u{00A0}'.into())),
@ -76,6 +74,7 @@ impl Markup {
&NodeKind::Quote { double } => Some(MarkupNode::Quote { double }),
NodeKind::Strong => node.cast().map(MarkupNode::Strong),
NodeKind::Emph => node.cast().map(MarkupNode::Emph),
NodeKind::Link(url) => Some(MarkupNode::Link(url.clone())),
NodeKind::Raw(raw) => Some(MarkupNode::Raw(raw.as_ref().clone())),
NodeKind::Math(math) => Some(MarkupNode::Math(Spanned::new(
math.as_ref().clone(),
@ -84,6 +83,7 @@ impl Markup {
NodeKind::Heading => node.cast().map(MarkupNode::Heading),
NodeKind::List => node.cast().map(MarkupNode::List),
NodeKind::Enum => node.cast().map(MarkupNode::Enum),
NodeKind::Desc => node.cast().map(MarkupNode::Desc),
NodeKind::Label(v) => Some(MarkupNode::Label(v.clone())),
NodeKind::Ref(v) => Some(MarkupNode::Ref(v.clone())),
_ => node.cast().map(MarkupNode::Expr),
@ -96,8 +96,8 @@ impl Markup {
pub enum MarkupNode {
/// Whitespace containing less than two newlines.
Space,
/// A forced line break: `\` or `\+` if justified.
Linebreak { justified: bool },
/// A forced line break.
Linebreak,
/// A paragraph break: Two or more newlines.
Parbreak,
/// Plain text.
@ -108,6 +108,8 @@ pub enum MarkupNode {
Strong(StrongNode),
/// Emphasized content: `_Emphasized_`.
Emph(EmphNode),
/// A hyperlink.
Link(EcoString),
/// A raw block with optional syntax highlighting: `` `...` ``.
Raw(RawNode),
/// A math formula: `$a^2 = b^2 + c^2$`.
@ -116,8 +118,10 @@ pub enum MarkupNode {
Heading(HeadingNode),
/// An item in an unordered list: `- ...`.
List(ListNode),
/// An item in an enumeration (ordered list): `1. ...`.
/// An item in an enumeration (ordered list): `+ ...` or `1. ...`.
Enum(EnumNode),
/// An item in a description list: `/ Term: Details.
Desc(DescNode),
/// A label.
Label(EcoString),
/// A reference.
@ -170,8 +174,8 @@ pub struct RawNode {
pub struct MathNode {
/// The formula between the dollars / brackets.
pub formula: EcoString,
/// Whether the formula is display-level, that is, it is surrounded by
/// `$[..]$`.
/// Whether the formula is display-level, that is, it contains whitespace
/// after the starting dollar sign and before the ending dollar sign.
pub display: bool,
}
@ -205,7 +209,7 @@ node! {
impl ListNode {
/// The contents of the list item.
pub fn body(&self) -> Markup {
self.0.cast_first_child().expect("list node is missing body")
self.0.cast_first_child().expect("list item is missing body")
}
}
@ -217,18 +221,36 @@ node! {
impl EnumNode {
/// The contents of the list item.
pub fn body(&self) -> Markup {
self.0.cast_first_child().expect("enum node is missing body")
self.0.cast_first_child().expect("enum item is missing body")
}
/// The number, if any.
pub fn number(&self) -> Option<usize> {
self.0.children().find_map(|node| match node.kind() {
NodeKind::EnumNumbering(num) => Some(*num),
_ => None,
})
}
}
node! {
/// An item in a description list: `/ Term: Details.
DescNode: Desc
}
impl DescNode {
/// The term described by the list item.
pub fn term(&self) -> Markup {
self.0
.children()
.find_map(|node| match node.kind() {
NodeKind::EnumNumbering(num) => Some(*num),
_ => None,
})
.expect("enum node is missing number")
.cast_first_child()
.expect("description list item is missing term")
}
/// The description of the term.
pub fn body(&self) -> Markup {
self.0
.cast_last_child()
.expect("description list item is missing body")
}
}

144
src/syntax/highlight.rs

@ -147,12 +147,12 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String {
/// The syntax highlighting category of a node.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum Category {
/// A line or block comment.
Comment,
/// Any kind of bracket, parenthesis or brace.
Bracket,
/// Punctuation in code.
Punctuation,
/// A line or block comment.
Comment,
/// An easily typable shortcut to a unicode codepoint.
Shortcut,
/// An escape sequence.
@ -161,14 +161,18 @@ pub enum Category {
Strong,
/// Emphasized text.
Emph,
/// A hyperlink.
Link,
/// Raw text or code.
Raw,
/// A math formula.
Math,
/// A section heading.
Heading,
/// A list or enumeration.
/// A symbol of a list, enumeration, or description list.
List,
/// A term in a description list.
Term,
/// A label.
Label,
/// A reference.
@ -204,66 +208,74 @@ impl Category {
i: usize,
) -> Option<Category> {
match child.kind() {
NodeKind::LineComment => Some(Category::Comment),
NodeKind::BlockComment => Some(Category::Comment),
NodeKind::LeftBrace => Some(Category::Bracket),
NodeKind::RightBrace => Some(Category::Bracket),
NodeKind::LeftBracket => Some(Category::Bracket),
NodeKind::RightBracket => Some(Category::Bracket),
NodeKind::LeftParen => Some(Category::Bracket),
NodeKind::RightParen => Some(Category::Bracket),
NodeKind::Comma => Some(Category::Punctuation),
NodeKind::Semicolon => Some(Category::Punctuation),
NodeKind::Colon => Some(Category::Punctuation),
NodeKind::Dot => Some(Category::Punctuation),
NodeKind::LineComment => Some(Category::Comment),
NodeKind::BlockComment => Some(Category::Comment),
NodeKind::Markup { .. } => match parent.kind() {
NodeKind::Desc
if parent
.children()
.take_while(|child| child.kind() != &NodeKind::Colon)
.find(|c| matches!(c.kind(), NodeKind::Markup { .. }))
.map_or(false, |ident| std::ptr::eq(ident, child)) =>
{
Some(Category::Term)
}
_ => None,
},
NodeKind::Space { .. } => None,
NodeKind::Linebreak { .. } => Some(Category::Shortcut),
NodeKind::Text(_) => None,
NodeKind::Escape(_) => Some(Category::Escape),
NodeKind::NonBreakingSpace => Some(Category::Shortcut),
NodeKind::Shy => Some(Category::Shortcut),
NodeKind::EnDash => Some(Category::Shortcut),
NodeKind::EmDash => Some(Category::Shortcut),
NodeKind::Ellipsis => Some(Category::Shortcut),
NodeKind::Escape(_) => Some(Category::Escape),
NodeKind::Strong => Some(Category::Strong),
NodeKind::Emph => Some(Category::Emph),
NodeKind::Raw(_) => Some(Category::Raw),
NodeKind::Math(_) => Some(Category::Math),
NodeKind::Heading => Some(Category::Heading),
NodeKind::Minus => match parent.kind() {
NodeKind::List => Some(Category::List),
_ => Some(Category::Operator),
},
NodeKind::EnumNumbering(_) => Some(Category::List),
NodeKind::Label(_) => Some(Category::Label),
NodeKind::Ref(_) => Some(Category::Ref),
NodeKind::Not => Some(Category::Keyword),
NodeKind::And => Some(Category::Keyword),
NodeKind::Or => Some(Category::Keyword),
NodeKind::Let => Some(Category::Keyword),
NodeKind::Set => Some(Category::Keyword),
NodeKind::Show => Some(Category::Keyword),
NodeKind::Wrap => Some(Category::Keyword),
NodeKind::If => Some(Category::Keyword),
NodeKind::Else => Some(Category::Keyword),
NodeKind::While => Some(Category::Keyword),
NodeKind::For => Some(Category::Keyword),
NodeKind::In => Some(Category::Keyword),
NodeKind::As => Some(Category::Keyword),
NodeKind::Break => Some(Category::Keyword),
NodeKind::Continue => Some(Category::Keyword),
NodeKind::Return => Some(Category::Keyword),
NodeKind::Import => Some(Category::Keyword),
NodeKind::From => Some(Category::Keyword),
NodeKind::Include => Some(Category::Keyword),
NodeKind::Plus => Some(Category::Operator),
NodeKind::Quote { .. } => None,
NodeKind::Star => match parent.kind() {
NodeKind::Strong => None,
_ => Some(Category::Operator),
},
NodeKind::Slash => Some(Category::Operator),
NodeKind::PlusEq => Some(Category::Operator),
NodeKind::HyphEq => Some(Category::Operator),
NodeKind::StarEq => Some(Category::Operator),
NodeKind::SlashEq => Some(Category::Operator),
NodeKind::Underscore => None,
NodeKind::Strong => Some(Category::Strong),
NodeKind::Emph => Some(Category::Emph),
NodeKind::Link(_) => Some(Category::Link),
NodeKind::Raw(_) => Some(Category::Raw),
NodeKind::Math(_) => Some(Category::Math),
NodeKind::Heading => Some(Category::Heading),
NodeKind::List => None,
NodeKind::Enum => None,
NodeKind::EnumNumbering(_) => Some(Category::List),
NodeKind::Desc => None,
NodeKind::Label(_) => Some(Category::Label),
NodeKind::Ref(_) => Some(Category::Ref),
NodeKind::Comma => Some(Category::Punctuation),
NodeKind::Semicolon => Some(Category::Punctuation),
NodeKind::Colon => match parent.kind() {
NodeKind::Desc => Some(Category::Term),
_ => Some(Category::Punctuation),
},
NodeKind::Plus => match parent.kind() {
NodeKind::Enum => Some(Category::List),
_ => Some(Category::Operator),
},
NodeKind::Minus => match parent.kind() {
NodeKind::List => Some(Category::List),
_ => Some(Category::Operator),
},
NodeKind::Slash => match parent.kind() {
NodeKind::Desc => Some(Category::List),
_ => Some(Category::Operator),
},
NodeKind::Dot => Some(Category::Punctuation),
NodeKind::Eq => match parent.kind() {
NodeKind::Heading => None,
_ => Some(Category::Operator),
@ -274,10 +286,34 @@ impl Category {
NodeKind::LtEq => Some(Category::Operator),
NodeKind::Gt => Some(Category::Operator),
NodeKind::GtEq => Some(Category::Operator),
NodeKind::PlusEq => Some(Category::Operator),
NodeKind::HyphEq => Some(Category::Operator),
NodeKind::StarEq => Some(Category::Operator),
NodeKind::SlashEq => Some(Category::Operator),
NodeKind::Dots => Some(Category::Operator),
NodeKind::Arrow => Some(Category::Operator),
NodeKind::Not => Some(Category::Keyword),
NodeKind::And => Some(Category::Keyword),
NodeKind::Or => Some(Category::Keyword),
NodeKind::None => Some(Category::None),
NodeKind::Auto => Some(Category::Auto),
NodeKind::Let => Some(Category::Keyword),
NodeKind::Set => Some(Category::Keyword),
NodeKind::Show => Some(Category::Keyword),
NodeKind::Wrap => Some(Category::Keyword),
NodeKind::If => Some(Category::Keyword),
NodeKind::Else => Some(Category::Keyword),
NodeKind::For => Some(Category::Keyword),
NodeKind::In => Some(Category::Keyword),
NodeKind::While => Some(Category::Keyword),
NodeKind::Break => Some(Category::Keyword),
NodeKind::Continue => Some(Category::Keyword),
NodeKind::Return => Some(Category::Keyword),
NodeKind::Import => Some(Category::Keyword),
NodeKind::Include => Some(Category::Keyword),
NodeKind::From => Some(Category::Keyword),
NodeKind::As => Some(Category::Keyword),
NodeKind::Ident(_) => match parent.kind() {
NodeKind::Markup { .. } => Some(Category::Interpolated),
NodeKind::FuncCall => Some(Category::Function),
@ -302,15 +338,6 @@ impl Category {
NodeKind::Float(_) => Some(Category::Number),
NodeKind::Numeric(_, _) => Some(Category::Number),
NodeKind::Str(_) => Some(Category::String),
NodeKind::Error(_, _) => Some(Category::Invalid),
NodeKind::Unknown(_) => Some(Category::Invalid),
NodeKind::Underscore => None,
NodeKind::Markup { .. } => None,
NodeKind::Space { .. } => None,
NodeKind::Text(_) => None,
NodeKind::Quote { .. } => None,
NodeKind::List => None,
NodeKind::Enum => None,
NodeKind::CodeBlock => None,
NodeKind::ContentBlock => None,
NodeKind::GroupExpr => None,
@ -341,6 +368,9 @@ impl Category {
NodeKind::BreakExpr => None,
NodeKind::ContinueExpr => None,
NodeKind::ReturnExpr => None,
NodeKind::Error(_, _) => Some(Category::Invalid),
NodeKind::Unknown(_) => Some(Category::Invalid),
}
}
@ -354,10 +384,12 @@ impl Category {
Self::Escape => "constant.character.escape.content.typst",
Self::Strong => "markup.bold.typst",
Self::Emph => "markup.italic.typst",
Self::Link => "markup.underline.link.typst",
Self::Raw => "markup.raw.typst",
Self::Math => "string.other.math.typst",
Self::Heading => "markup.heading.typst",
Self::List => "markup.list.typst",
Self::Term => "markup.bold.typst",
Self::Label => "entity.name.label.typst",
Self::Ref => "markup.other.reference.typst",
Self::Keyword => "keyword.typst",

272
src/syntax/mod.rs

@ -571,6 +571,14 @@ impl PartialEq for NodeData {
/// the parser.
#[derive(Debug, Clone, PartialEq)]
pub enum NodeKind {
/// A line comment, two slashes followed by inner contents, terminated with
/// a newline: `//<str>\n`.
LineComment,
/// A block comment, a slash and a star followed by inner contents,
/// terminated with a star and a slash: `/*<str>*/`.
///
/// The comment can contain nested block comments.
BlockComment,
/// A left curly brace, starting a code block: `{`.
LeftBrace,
/// A right curly brace, terminating a code block: `}`.
@ -585,23 +593,83 @@ pub enum NodeKind {
/// A right round parenthesis, terminating a grouped expression, collection,
/// argument or parameter list: `)`.
RightParen,
/// Markup of which all lines must have a minimal indentation.
///
/// Notably, the number does not determine in which column the markup
/// started, but to the right of which column all markup elements must be,
/// so it is zero except for headings and lists.
Markup { min_indent: usize },
/// One or more whitespace characters. Single spaces are collapsed into text
/// nodes if they would otherwise be surrounded by text nodes.
///
/// Also stores how many newlines are contained.
Space { newlines: usize },
/// A forced line break.
Linebreak,
/// Consecutive text without markup. While basic text with just single
/// spaces is collapsed into a single node, certain symbols that could
/// possibly be markup force text into multiple nodes.
Text(EcoString),
/// A slash and the letter "u" followed by a hexadecimal unicode entity
/// enclosed in curly braces: `\u{1F5FA}`.
Escape(char),
/// A non-breaking space: `~`.
NonBreakingSpace,
/// A soft hyphen: `-?`.
Shy,
/// An en-dash: `--`.
EnDash,
/// An em-dash: `---`.
EmDash,
/// An ellipsis: `...`.
Ellipsis,
/// A smart quote: `'` or `"`.
Quote { double: bool },
/// The strong text toggle, multiplication operator, and wildcard import
/// symbol: `*`.
Star,
/// Toggles emphasized text: `_`.
Underscore,
/// Strong content: `*Strong*`.
Strong,
/// Emphasized content: `_Emphasized_`.
Emph,
/// A hyperlink.
Link(EcoString),
/// A raw block with optional syntax highlighting: `` `...` ``.
Raw(Arc<RawNode>),
/// A math formula: `$x$`, `$[x^2]$`.
Math(Arc<MathNode>),
/// A section heading: `= Introduction`.
Heading,
/// An item in an unordered list: `- ...`.
List,
/// An item in an enumeration (ordered list): `+ ...` or `1. ...`.
Enum,
/// An explicit enumeration numbering: `23.`.
EnumNumbering(usize),
/// An item in a description list: `/ Term: Details.
Desc,
/// A label: `<label>`.
Label(EcoString),
/// A reference: `@label`.
Ref(EcoString),
/// A comma separator in a sequence: `,`.
Comma,
/// A semicolon terminating an expression: `;`.
Semicolon,
/// A colon between name / key and value in a dictionary, argument or
/// parameter list: `:`.
/// parameter list, or between the term and body of a description list
/// term: `:`.
Colon,
/// The unary plus and addition operator: `+`.
/// The unary plus and addition operator, and start of enum items: `+`.
Plus,
/// The unary negation and subtraction operator: `-`.
/// The unary negation and subtraction operator, and start of list
/// items: `-`.
Minus,
/// The division operator: `/`.
/// The division operator and start of description list items: `/`.
Slash,
/// A field access and method call operator: `.`.
Dot,
@ -627,16 +695,16 @@ pub enum NodeKind {
StarEq,
/// The divide-assign operator: `/=`.
SlashEq,
/// The spread operator: `..`.
Dots,
/// An arrow between a closure's parameters and body: `=>`.
Arrow,
/// The `not` operator.
Not,
/// The `and` operator.
And,
/// The `or` operator.
Or,
/// The spread operator: `..`.
Dots,
/// An arrow between a closure's parameters and body: `=>`.
Arrow,
/// The none literal: `none`.
None,
/// The auto literal: `auto`.
@ -673,60 +741,7 @@ pub enum NodeKind {
From,
/// The `as` keyword.
As,
/// Markup of which all lines must have a minimal indentation.
///
/// Notably, the number does not determine in which column the markup
/// started, but to the right of which column all markup elements must be,
/// so it is zero except for headings and lists.
Markup { min_indent: usize },
/// One or more whitespace characters. Single spaces are collapsed into text
/// nodes if they would otherwise be surrounded by text nodes.
///
/// Also stores how many newlines are contained.
Space { newlines: usize },
/// Consecutive text without markup. While basic text with just single
/// spaces is collapsed into a single node, certain symbols that could
/// possibly be markup force text into multiple nodes.
Text(EcoString),
/// A forced line break: `\` or `\+` if justified.
Linebreak { justified: bool },
/// A non-breaking space: `~`.
NonBreakingSpace,
/// A soft hyphen: `-?`.
Shy,
/// An en-dash: `--`.
EnDash,
/// An em-dash: `---`.
EmDash,
/// An ellipsis: `...`.
Ellipsis,
/// A smart quote: `'` or `"`.
Quote { double: bool },
/// A slash and the letter "u" followed by a hexadecimal unicode entity
/// enclosed in curly braces: `\u{1F5FA}`.
Escape(char),
/// Strong content: `*Strong*`.
Strong,
/// Emphasized content: `_Emphasized_`.
Emph,
/// A raw block with optional syntax highlighting: `` `...` ``.
Raw(Arc<RawNode>),
/// A math formula: `$x$`, `$[x^2]$`.
Math(Arc<MathNode>),
/// A section heading: `= Introduction`.
Heading,
/// An item in an unordered list: `- ...`.
List,
/// An item in an enumeration (ordered list): `1. ...`.
Enum,
/// A numbering: `23.`.
///
/// Can also exist without the number: `.`.
EnumNumbering(Option<usize>),
/// A label: `<label>`.
Label(EcoString),
/// A reference: `@label`.
Ref(EcoString),
/// An identifier: `center`.
Ident(EcoString),
/// A boolean: `true`, `false`.
@ -799,14 +814,7 @@ pub enum NodeKind {
ContinueExpr,
/// A return expression: `return x + 1`.
ReturnExpr,
/// A line comment, two slashes followed by inner contents, terminated with
/// a newline: `//<str>\n`.
LineComment,
/// A block comment, a slash and a star followed by inner contents,
/// terminated with a star and a slash: `/*<str>*/`.
///
/// The comment can contain nested block comments.
BlockComment,
/// Tokens that appear in the wrong place.
Error(SpanPos, EcoString),
/// Unknown character sequences.
@ -844,7 +852,7 @@ impl NodeKind {
}
}
/// Whether changes _inside_ this node are safely encapuslated, so that only
/// Whether changes _inside_ this node are safely encapsulated, so that only
/// this node must be reparsed.
pub fn is_bounded(&self) -> bool {
match self {
@ -860,7 +868,6 @@ impl NodeKind {
| Self::BlockComment
| Self::Space { .. }
| Self::Escape(_) => true,
Self::Text(t) => t != "-" && !t.ends_with('.'),
_ => false,
}
}
@ -868,14 +875,43 @@ impl NodeKind {
/// A human-readable name for the kind.
pub fn as_str(&self) -> &'static str {
match self {
Self::LineComment => "line comment",
Self::BlockComment => "block comment",
Self::LeftBrace => "opening brace",
Self::RightBrace => "closing brace",
Self::LeftBracket => "opening bracket",
Self::RightBracket => "closing bracket",
Self::LeftParen => "opening paren",
Self::RightParen => "closing paren",
Self::Markup { .. } => "markup",
Self::Space { newlines: (2 ..) } => "paragraph break",
Self::Space { .. } => "space",
Self::Linebreak => "linebreak",
Self::Text(_) => "text",
Self::Escape(_) => "escape sequence",
Self::NonBreakingSpace => "non-breaking space",
Self::Shy => "soft hyphen",
Self::EnDash => "en dash",
Self::EmDash => "em dash",
Self::Ellipsis => "ellipsis",
Self::Quote { double: false } => "single quote",
Self::Quote { double: true } => "double quote",
Self::Star => "star",
Self::Underscore => "underscore",
Self::Strong => "strong content",
Self::Emph => "emphasized content",
Self::Link(_) => "link",
Self::Raw(_) => "raw block",
Self::Math(_) => "math formula",
Self::Heading => "heading",
Self::List => "list item",
Self::Enum => "enumeration item",
Self::EnumNumbering(_) => "enumeration item numbering",
Self::Desc => "description list item",
Self::Label(_) => "label",
Self::Ref(_) => "reference",
Self::Comma => "comma",
Self::Semicolon => "semicolon",
Self::Colon => "colon",
@ -894,11 +930,11 @@ impl NodeKind {
Self::HyphEq => "subtract-assign operator",
Self::StarEq => "multiply-assign operator",
Self::SlashEq => "divide-assign operator",
Self::Dots => "dots",
Self::Arrow => "arrow",
Self::Not => "operator `not`",
Self::And => "operator `and`",
Self::Or => "operator `or`",
Self::Dots => "dots",
Self::Arrow => "arrow",
Self::None => "`none`",
Self::Auto => "`auto`",
Self::Let => "keyword `let`",
@ -909,7 +945,6 @@ impl NodeKind {
Self::Else => "keyword `else`",
Self::For => "keyword `for`",
Self::In => "keyword `in`",
Self::As => "keyword `as`",
Self::While => "keyword `while`",
Self::Break => "keyword `break`",
Self::Continue => "keyword `continue`",
@ -917,30 +952,8 @@ impl NodeKind {
Self::Import => "keyword `import`",
Self::Include => "keyword `include`",
Self::From => "keyword `from`",
Self::Markup { .. } => "markup",
Self::Space { newlines: (2 ..) } => "paragraph break",
Self::Space { .. } => "space",
Self::Linebreak { justified: false } => "linebreak",
Self::Linebreak { justified: true } => "justified linebreak",
Self::Text(_) => "text",
Self::NonBreakingSpace => "non-breaking space",
Self::Shy => "soft hyphen",
Self::EnDash => "en dash",
Self::EmDash => "em dash",
Self::Ellipsis => "ellipsis",
Self::Quote { double: false } => "single quote",
Self::Quote { double: true } => "double quote",
Self::Escape(_) => "escape sequence",
Self::Strong => "strong content",
Self::Emph => "emphasized content",
Self::Raw(_) => "raw block",
Self::Math(_) => "math formula",
Self::List => "list item",
Self::Heading => "heading",
Self::Enum => "enumeration item",
Self::EnumNumbering(_) => "enumeration item numbering",
Self::Label(_) => "label",
Self::Ref(_) => "reference",
Self::As => "keyword `as`",
Self::Ident(_) => "identifier",
Self::Bool(_) => "boolean",
Self::Int(_) => "integer",
@ -977,8 +990,7 @@ impl NodeKind {
Self::BreakExpr => "`break` expression",
Self::ContinueExpr => "`continue` expression",
Self::ReturnExpr => "`return` expression",
Self::LineComment => "line comment",
Self::BlockComment => "block comment",
Self::Error(_, _) => "parse error",
Self::Unknown(text) => match text.as_str() {
"*/" => "end of block comment",
@ -998,14 +1010,41 @@ impl Hash for NodeKind {
fn hash<H: Hasher>(&self, state: &mut H) {
std::mem::discriminant(self).hash(state);
match self {
Self::LineComment => {}
Self::BlockComment => {}
Self::LeftBrace => {}
Self::RightBrace => {}
Self::LeftBracket => {}
Self::RightBracket => {}
Self::LeftParen => {}
Self::RightParen => {}
Self::Markup { min_indent } => min_indent.hash(state),
Self::Space { newlines } => newlines.hash(state),
Self::Linebreak => {}
Self::Text(s) => s.hash(state),
Self::Escape(c) => c.hash(state),
Self::NonBreakingSpace => {}
Self::Shy => {}
Self::EnDash => {}
Self::EmDash => {}
Self::Ellipsis => {}
Self::Quote { double } => double.hash(state),
Self::Star => {}
Self::Underscore => {}
Self::Strong => {}
Self::Emph => {}
Self::Link(link) => link.hash(state),
Self::Raw(raw) => raw.hash(state),
Self::Math(math) => math.hash(state),
Self::Heading => {}
Self::List => {}
Self::Enum => {}
Self::EnumNumbering(num) => num.hash(state),
Self::Desc => {}
Self::Label(c) => c.hash(state),
Self::Ref(c) => c.hash(state),
Self::Comma => {}
Self::Semicolon => {}
Self::Colon => {}
@ -1024,11 +1063,11 @@ impl Hash for NodeKind {
Self::HyphEq => {}
Self::StarEq => {}
Self::SlashEq => {}
Self::Dots => {}
Self::Arrow => {}
Self::Not => {}
Self::And => {}
Self::Or => {}
Self::Dots => {}
Self::Arrow => {}
Self::None => {}
Self::Auto => {}
Self::Let => {}
@ -1039,7 +1078,6 @@ impl Hash for NodeKind {
Self::Else => {}
Self::For => {}
Self::In => {}
Self::As => {}
Self::While => {}
Self::Break => {}
Self::Continue => {}
@ -1047,27 +1085,8 @@ impl Hash for NodeKind {
Self::Import => {}
Self::Include => {}
Self::From => {}
Self::Markup { min_indent } => min_indent.hash(state),
Self::Space { newlines } => newlines.hash(state),
Self::Linebreak { justified } => justified.hash(state),
Self::Text(s) => s.hash(state),
Self::NonBreakingSpace => {}
Self::Shy => {}
Self::EnDash => {}
Self::EmDash => {}
Self::Ellipsis => {}
Self::Quote { double } => double.hash(state),
Self::Escape(c) => c.hash(state),
Self::Strong => {}
Self::Emph => {}
Self::Raw(raw) => raw.hash(state),
Self::Math(math) => math.hash(state),
Self::List => {}
Self::Heading => {}
Self::Enum => {}
Self::EnumNumbering(num) => num.hash(state),
Self::Label(c) => c.hash(state),
Self::Ref(c) => c.hash(state),
Self::As => {}
Self::Ident(v) => v.hash(state),
Self::Bool(v) => v.hash(state),
Self::Int(v) => v.hash(state),
@ -1104,8 +1123,7 @@ impl Hash for NodeKind {
Self::BreakExpr => {}
Self::ContinueExpr => {}
Self::ReturnExpr => {}
Self::LineComment => {}
Self::BlockComment => {}
Self::Error(pos, msg) => (pos, msg).hash(state),
Self::Unknown(text) => text.hash(state),
}

BIN
tests/ref/code/comment.png

Binary file not shown.
Side by Side Swipe Overlay

Before

(image error) Size: 2.6 KiB

After

(image error) Size: 797 B

BIN
tests/ref/structure/desc.png Normal file

Binary file not shown.
Side by Side

After

(image error) Size: 16 KiB

BIN
tests/ref/structure/enum.png

Binary file not shown.
Side by Side Swipe

Before

(image error) Size: 27 KiB

After

(image error) Size: 28 KiB

BIN
tests/ref/style/show-text.png

Binary file not shown.
Side by Side Swipe

Before

(image error) Size: 44 KiB

After

(image error) Size: 44 KiB

BIN
tests/ref/text/escape.png

Binary file not shown.
Side by Side Swipe

Before

(image error) Size: 14 KiB

After

(image error) Size: 13 KiB

BIN
tests/ref/text/link.png

Binary file not shown.
Side by Side Swipe

Before

(image error) Size: 40 KiB

After

(image error) Size: 48 KiB

5
tests/typ/code/comment.typ

@ -24,11 +24,6 @@ Still comment.
E
---
// Line comments have a special case for URLs.
https://example.com \
https:/* block comments don't ... */
---
// End should not appear without start.
// Error: 7-9 unexpected end of block comment

10
tests/typ/math/basic.typ

@ -5,15 +5,15 @@ The sum of $a$ and $b$ is $a + b$.
---
We will show that:
$[ a^2 + b^2 = c^2 ]$
$ a^2 + b^2 = c^2 $
---
Prove by induction:
$[ \sum_{k=0}^n k = \frac{n(n+1)}{2} ]$
$ \sum_{k=0}^n k = \frac{n(n+1)}{2} $
---
// Test that blackboard style looks nice.
$[ f: \mathbb{N} \rightarrow \mathbb{R} ]$
$ f: \mathbb{N} \rightarrow \mathbb{R} $
---
#set math(family: "IBM Plex Sans")
@ -26,5 +26,5 @@ $a$
$\sqrt{x$
---
// Error: 2:1 expected closing bracket and dollar sign
$[a
// Error: 2:1 expected dollar sign
$a

48
tests/typ/structure/desc.typ Normal file

@ -0,0 +1,48 @@
// Test description lists.
---
/
No: list \
/No: list
---
// Test with constructor.
#desc(
(term: [One], body: [First]),
(term: [Two], body: [Second]),
)
---
// Test joining.
#for word in lorem(4).split().map(s => s.trim(".")) [
/ #word: Latin stuff.
]
---
// Test multiline.
#set text(8pt)
/ Fruit: A tasty, edible thing.
/ Veggie:
An important energy source
for vegetarians.
---
// Test style change.
#set text(8pt)
/ First list: #lorem(4)
#set desc(body-indent: 30pt)
/ Second list: #lorem(4)
---
// Test grid like show rule.
#show it: desc as table(
columns: 2,
padding: 3pt,
..it.items.map(item => (emph(item.term), item.body)).flatten(),
)
/ A: One letter
/ BB: Two letters
/ CCC: Three letters

29
tests/typ/structure/enum.typ

@ -1,4 +1,4 @@
// Test enums.
// Test enumerations.
---
#enum[Embrace][Extend][Extinguish]
@ -12,28 +12,28 @@
---
2. Second
1. First
. Indented
+ Indented
---
// Test automatic numbering in summed content.
#for i in range(5) {
[. #roman(1 + i)]
[+ #roman(1 + i)]
}
---
// Test label pattern.
#set enum(label: "~ A:")
. First
. Second
1. First
+ Second
#set enum(label: "(*)")
. A
. B
. C
+ A
+ B
+ C
#set enum(label: "i)")
. A
. B
+ A
+ B
---
// Test label closure.
@ -47,12 +47,13 @@
---
#set enum(label: n => n > 1)
. A
. B
+ A
+ B
---
// Lone dot is not a list.
.
// Lone plus is not an enum.
+
No enum
---
// Error: 18-20 invalid pattern

2
tests/typ/structure/list.typ

@ -1,4 +1,4 @@
// Test lists.
// Test unordered lists.
---
-

6
tests/typ/style/set.typ

@ -44,9 +44,9 @@ Hello *{x}*
move(dy: -0.15em, image(path, width: 1em, height: 1em))
})
. Monkey
. Rhino
. Tiger
+ Monkey
+ Rhino
+ Tiger
---
// Error: 11-25 set is only allowed directly in code and content blocks

7
tests/typ/text/escape.typ

@ -2,15 +2,12 @@
---
// Escapable symbols.
\\ \/ \[ \] \{ \} \# \* \_ \
\= \~ \` \$ \" \' \< \> \@
\\ \/ \[ \] \{ \} \# \* \_ \+ \= \~ \
\` \$ \" \' \< \> \@ \( \) \A
// No need to escape.
( ) ;
// Unescapable.
\a \: \; \( \)
// Escaped comments.
\//
\/\* \*\/

4
tests/typ/text/justify.typ

@ -21,8 +21,8 @@ D
---
// Test forced justification with justified break.
A B C \+
D E F \+
A B C #linebreak(justify: true)
D E F #linebreak(justify: true)
---
// Test that justificating chinese text is at least a bit sensible.

6
tests/typ/text/linebreak.typ

@ -30,7 +30,7 @@ Trailing break \ \
---
// Test justified breaks.
#set par(justify: true)
With a soft \+
break you can force a break without #linebreak(justified: true)
breaking justification. #linebreak(justified: false)
With a soft #linebreak(justify: true)
break you can force a break without #linebreak(justify: true)
breaking justification. #linebreak(justify: false)
Nice!

13
tests/typ/text/link.typ

@ -1,8 +1,8 @@
// Test hyperlinking.
---
// Link without body.
#link("https://example.com/")
// Link syntax.
https://example.com/
// Link with body.
#link("https://typst.app/")[Some text text text]
@ -10,10 +10,17 @@
// With line break.
This link appears #link("https://google.com/")[in the middle of] a paragraph.
// Prefix is trimmed.
// Certain prefixes are trimmed when using the `link` function.
Contact #link("mailto:hi@typst.app") or
call #link("tel:123") for more information.
---
// Test that the period is trimmed.
https://a.b.?q=%10#. \
Wahttp://link \
Nohttps:\//link \
Nohttp\://comment
---
// Styled with underline and color.
#set link(fill: rgb("283663"))

2
tests/typ/text/par.typ

@ -39,7 +39,7 @@ Hello
fn main() {}
```
$[ x + y = z ]$
$ x + y = z $
- List