diff --git a/src/eval/mod.rs b/src/eval/mod.rs index fb65420d0..7ef5bf764 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -195,16 +195,20 @@ impl Eval for MarkupNode { Ok(match self { Self::Space => Content::Space, Self::Parbreak => Content::Parbreak, - &Self::Linebreak { justified } => Content::Linebreak { justified }, + &Self::Linebreak => Content::Linebreak { justify: false }, Self::Text(text) => Content::Text(text.clone()), &Self::Quote { double } => Content::Quote { double }, Self::Strong(strong) => strong.eval(vm)?, Self::Emph(emph) => emph.eval(vm)?, + Self::Link(url) => { + Content::show(library::text::LinkNode::from_url(url.clone())) + } Self::Raw(raw) => raw.eval(vm)?, Self::Math(math) => math.eval(vm)?, Self::Heading(heading) => heading.eval(vm)?, Self::List(list) => list.eval(vm)?, Self::Enum(enum_) => enum_.eval(vm)?, + Self::Desc(desc) => desc.eval(vm)?, Self::Label(_) => Content::Empty, Self::Ref(label) => Content::show(library::structure::RefNode(label.clone())), Self::Expr(expr) => expr.eval(vm)?.display(), @@ -273,11 +277,8 @@ impl Eval for ListNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult { - Ok(Content::Item(library::structure::ListItem { - kind: library::structure::UNORDERED, - number: None, - body: Box::new(self.body().eval(vm)?), - })) + let body = Box::new(self.body().eval(vm)?); + Ok(Content::Item(library::structure::ListItem::List(body))) } } @@ -285,11 +286,23 @@ impl Eval for EnumNode { type Output = Content; fn eval(&self, vm: &mut Vm) -> SourceResult { - Ok(Content::Item(library::structure::ListItem { - kind: library::structure::ORDERED, - number: self.number(), - body: Box::new(self.body().eval(vm)?), - })) + let number = self.number(); + let body = Box::new(self.body().eval(vm)?); + Ok(Content::Item(library::structure::ListItem::Enum( + number, body, + ))) + } +} + +impl Eval for DescNode { + type Output = Content; + + fn eval(&self, vm: &mut Vm) -> SourceResult { + let term = self.term().eval(vm)?; + let body = self.body().eval(vm)?; + Ok(Content::Item(library::structure::ListItem::Desc(Box::new( + library::structure::DescItem { term, body }, + )))) } } diff --git a/src/library/mod.rs b/src/library/mod.rs index 91e4671c5..e7617bc0c 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -41,6 +41,7 @@ pub fn new() -> Scope { std.def_node::("heading"); std.def_node::("list"); std.def_node::("enum"); + std.def_node::("desc"); std.def_node::("table"); // Layout. diff --git a/src/library/structure/list.rs b/src/library/structure/list.rs index 9da147338..7a43e5db1 100644 --- a/src/library/structure/list.rs +++ b/src/library/structure/list.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use unscanny::Scanner; use crate::library::layout::{BlockSpacing, GridNode, TrackSizing}; @@ -9,9 +7,7 @@ use crate::library::utility::Numbering; /// An unordered (bulleted) or ordered (numbered) list. #[derive(Debug, Hash)] -pub struct ListNode { - /// Where the list starts. - pub start: usize, +pub struct ListNode { /// If true, the items are separated by leading instead of list spacing. pub tight: bool, /// If true, the spacing above the list is leading instead of above spacing. @@ -20,19 +16,11 @@ pub struct ListNode { pub items: StyleVec, } -/// An item in a list. -#[derive(Clone, PartialEq, Hash)] -pub struct ListItem { - /// The kind of item. - pub kind: ListKind, - /// The number of the item. - pub number: Option, - /// The node that produces the item's body. - pub body: Box, -} - /// An ordered list. -pub type EnumNode = ListNode; +pub type EnumNode = ListNode; + +/// A description list. +pub type DescNode = ListNode; #[node(showable)] impl ListNode { @@ -44,7 +32,11 @@ impl ListNode { pub const INDENT: RawLength = RawLength::zero(); /// The space between the label and the body of each item. #[property(resolve)] - pub const BODY_INDENT: RawLength = Em::new(0.5).into(); + pub const BODY_INDENT: RawLength = Em::new(match L { + LIST | ENUM => 0.5, + DESC | _ => 1.0, + }) + .into(); /// The spacing above the list. #[property(resolve, shorthand(around))] @@ -57,19 +49,34 @@ impl ListNode { pub const SPACING: BlockSpacing = Ratio::one().into(); fn construct(_: &mut Vm, args: &mut Args) -> SourceResult { - Ok(Content::show(Self { - start: args.named("start")?.unwrap_or(1), - tight: args.named("tight")?.unwrap_or(true), - attached: args.named("attached")?.unwrap_or(false), - items: args + let items = match L { + LIST => args .all()? .into_iter() - .map(|body| ListItem { - kind: L, - number: None, - body: Box::new(body), - }) + .map(|body| ListItem::List(Box::new(body))) .collect(), + ENUM => { + let mut number: usize = args.named("start")?.unwrap_or(1); + args.all()? + .into_iter() + .map(|body| { + let item = ListItem::Enum(Some(number), Box::new(body)); + number += 1; + item + }) + .collect() + } + DESC | _ => args + .all()? + .into_iter() + .map(|item| ListItem::Desc(Box::new(item))) + .collect(), + }; + + Ok(Content::show(Self { + tight: args.named("tight")?.unwrap_or(true), + attached: args.named("attached")?.unwrap_or(false), + items, })) } } @@ -77,10 +84,7 @@ impl ListNode { impl Show for ListNode { fn unguard(&self, sel: Selector) -> ShowNode { Self { - items: self.items.map(|item| ListItem { - body: Box::new(item.body.unguard(sel).role(Role::ListItemBody)), - ..*item - }), + items: self.items.map(|item| item.unguard(sel)), ..*self } .pack() @@ -88,13 +92,12 @@ impl Show for ListNode { fn encode(&self, _: StyleChain) -> Dict { dict! { - "start" => Value::Int(self.start as i64), "tight" => Value::Bool(self.tight), "attached" => Value::Bool(self.attached), "items" => Value::Array( self.items .items() - .map(|item| Value::Content(item.body.as_ref().clone())) + .map(|item| item.encode()) .collect() ), } @@ -106,34 +109,54 @@ impl Show for ListNode { styles: StyleChain, ) -> SourceResult { let mut cells = vec![]; - let mut number = self.start; + let mut number = 1; let label = styles.get(Self::LABEL); - - for (item, map) in self.items.iter() { - number = item.number.unwrap_or(number); - - cells.push(LayoutNode::default()); - cells.push( - label - .resolve(world, L, number)? - .styled_with_map(map.clone()) - .role(Role::ListLabel) - .pack(), - ); - cells.push(LayoutNode::default()); - cells.push((*item.body).clone().styled_with_map(map.clone()).pack()); - number += 1; - } - + let indent = styles.get(Self::INDENT); + let body_indent = styles.get(Self::BODY_INDENT); let gutter = if self.tight { styles.get(ParNode::LEADING) } else { styles.get(Self::SPACING) }; - let indent = styles.get(Self::INDENT); - let body_indent = styles.get(Self::BODY_INDENT); + for (item, map) in self.items.iter() { + if let &ListItem::Enum(Some(n), _) = item { + number = n; + } + + cells.push(LayoutNode::default()); + + let label = if L == LIST || L == ENUM { + label + .resolve(world, L, number)? + .styled_with_map(map.clone()) + .role(Role::ListLabel) + .pack() + } else { + LayoutNode::default() + }; + + cells.push(label); + cells.push(LayoutNode::default()); + + let body = match &item { + ListItem::List(body) => body.as_ref().clone(), + ListItem::Enum(_, body) => body.as_ref().clone(), + ListItem::Desc(item) => Content::sequence(vec![ + Content::Horizontal { + amount: (-body_indent).into(), + weak: false, + }, + (item.term.clone() + Content::Text(':'.into())).strong(), + Content::Space, + item.body.clone(), + ]), + }; + + cells.push(body.styled_with_map(map.clone()).pack()); + number += 1; + } Ok(Content::block(GridNode { tracks: Spec::with_x(vec![ @@ -165,35 +188,110 @@ impl Show for ListNode { } } - Ok(realized - .role(Role::List { ordered: L == ORDERED }) - .spaced(above, below)) + Ok(realized.role(Role::List { ordered: L == ENUM }).spaced(above, below)) + } +} + +/// An item in a list. +#[derive(Clone, PartialEq, Hash)] +pub enum ListItem { + /// An item of an unordered list. + List(Box), + /// An item of an ordered list. + Enum(Option, Box), + /// An item of a description list. + Desc(Box), +} + +impl ListItem { + /// What kind of item this is. + pub fn kind(&self) -> ListKind { + match self { + Self::List(_) => LIST, + Self::Enum { .. } => ENUM, + Self::Desc { .. } => DESC, + } + } + + fn unguard(&self, sel: Selector) -> Self { + match self { + Self::List(body) => Self::List(Box::new(body.unguard(sel))), + Self::Enum(number, body) => Self::Enum(*number, Box::new(body.unguard(sel))), + Self::Desc(item) => Self::Desc(Box::new(DescItem { + term: item.term.unguard(sel), + body: item.body.unguard(sel), + })), + } + } + + /// Encode the item into a value. + fn encode(&self) -> Value { + match self { + Self::List(body) => Value::Content(body.as_ref().clone()), + Self::Enum(number, body) => Value::Dict(dict! { + "number" => match *number { + Some(n) => Value::Int(n as i64), + None => Value::None, + }, + "body" => Value::Content(body.as_ref().clone()), + }), + Self::Desc(item) => Value::Dict(dict! { + "term" => Value::Content(item.term.clone()), + "body" => Value::Content(item.body.clone()), + }), + } } } impl Debug for ListItem { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - if self.kind == UNORDERED { - f.write_char('-')?; - } else { - if let Some(number) = self.number { - write!(f, "{}", number)?; - } - f.write_char('.')?; + match self { + Self::List(body) => write!(f, "- {body:?}"), + Self::Enum(number, body) => match number { + Some(n) => write!(f, "{n}. {body:?}"), + None => write!(f, "+ {body:?}"), + }, + Self::Desc(item) => item.fmt(f), } - f.write_char(' ')?; - self.body.fmt(f) + } +} + +/// A description list item. +#[derive(Clone, PartialEq, Hash)] +pub struct DescItem { + /// The term described by the list item. + pub term: Content, + /// The description of the term. + pub body: Content, +} + +castable! { + DescItem, + Expected: "dictionary with `term` and `body` keys", + Value::Dict(dict) => { + let term: Content = dict.get("term")?.clone().cast()?; + let body: Content = dict.get("body")?.clone().cast()?; + Self { term, body } + }, +} + +impl Debug for DescItem { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "/ {:?}: {:?}", self.term, self.body) } } /// How to label a list. pub type ListKind = usize; -/// Unordered list labelling style. -pub const UNORDERED: ListKind = 0; +/// An unordered list. +pub const LIST: ListKind = 0; -/// Ordered list labelling style. -pub const ORDERED: ListKind = 1; +/// An ordered list. +pub const ENUM: ListKind = 1; + +/// A description list. +pub const DESC: ListKind = 2; /// How to label a list or enumeration. #[derive(Debug, Clone, PartialEq, Hash)] @@ -218,8 +316,9 @@ impl Label { ) -> SourceResult { Ok(match self { Self::Default => match kind { - UNORDERED => Content::Text('•'.into()), - ORDERED | _ => Content::Text(format_eco!("{}.", number)), + LIST => Content::Text('•'.into()), + ENUM => Content::Text(format_eco!("{}.", number)), + DESC | _ => panic!("description lists don't have a label"), }, Self::Pattern(prefix, numbering, upper, suffix) => { let fmt = numbering.apply(number); diff --git a/src/library/text/link.rs b/src/library/text/link.rs index c06fea55d..78ae4d231 100644 --- a/src/library/text/link.rs +++ b/src/library/text/link.rs @@ -10,6 +10,13 @@ pub struct LinkNode { pub body: Option, } +impl LinkNode { + /// Create a link node from a URL with its bare text. + pub fn from_url(url: EcoString) -> Self { + Self { dest: Destination::Url(url), body: None } + } +} + #[node(showable)] impl LinkNode { /// The fill color of text in the link. Just the surrounding text color diff --git a/src/library/text/par.rs b/src/library/text/par.rs index 6910c23ac..859a7c87d 100644 --- a/src/library/text/par.rs +++ b/src/library/text/par.rs @@ -181,8 +181,8 @@ pub struct LinebreakNode; #[node] impl LinebreakNode { fn construct(_: &mut Vm, args: &mut Args) -> SourceResult { - let justified = args.named("justified")?.unwrap_or(false); - Ok(Content::Linebreak { justified }) + let justify = args.named("justify")?.unwrap_or(false); + Ok(Content::Linebreak { justify }) } } diff --git a/src/library/text/raw.rs b/src/library/text/raw.rs index e7c73a913..a64b1a924 100644 --- a/src/library/text/raw.rs +++ b/src/library/text/raw.rs @@ -91,7 +91,7 @@ impl Show for RawNode { let mut highlighter = HighlightLines::new(syntax, &THEME); for (i, line) in self.text.lines().enumerate() { if i != 0 { - seq.push(Content::Linebreak { justified: false }); + seq.push(Content::Linebreak { justify: false }); } for (style, piece) in diff --git a/src/model/content.rs b/src/model/content.rs index 92d592a6a..ae074b50e 100644 --- a/src/model/content.rs +++ b/src/model/content.rs @@ -14,7 +14,7 @@ use super::{ use crate::diag::StrResult; use crate::library::layout::{FlowChild, FlowNode, PageNode, PlaceNode, Spacing}; use crate::library::prelude::*; -use crate::library::structure::{DocNode, ListItem, ListNode, ORDERED, UNORDERED}; +use crate::library::structure::{DocNode, ListItem, ListNode, DESC, ENUM, LIST}; use crate::library::text::{ DecoNode, EmphNode, ParChild, ParNode, StrongNode, UNDERLINE, }; @@ -62,7 +62,7 @@ pub enum Content { /// A word space. Space, /// A forced line break. - Linebreak { justified: bool }, + Linebreak { justify: bool }, /// Horizontal spacing. Horizontal { amount: Spacing, weak: bool }, /// Plain text. @@ -264,7 +264,7 @@ impl Debug for Content { match self { Self::Empty => f.pad("Empty"), Self::Space => f.pad("Space"), - Self::Linebreak { justified } => write!(f, "Linebreak({justified})"), + Self::Linebreak { justify } => write!(f, "Linebreak({justify})"), Self::Horizontal { amount, weak } => { write!(f, "Horizontal({amount:?}, {weak})") } @@ -633,8 +633,8 @@ impl<'a> ParBuilder<'a> { Content::Space => { self.0.weak(ParChild::Text(' '.into()), styles, 2); } - &Content::Linebreak { justified } => { - let c = if justified { '\u{2028}' } else { '\n' }; + &Content::Linebreak { justify } => { + let c = if justify { '\u{2028}' } else { '\n' }; self.0.destructive(ParChild::Text(c.into()), styles); } &Content::Horizontal { amount, weak } => { @@ -734,7 +734,7 @@ impl<'a> ListBuilder<'a> { .items .items() .next() - .map_or(true, |first| item.kind == first.kind) => + .map_or(true, |first| item.kind() == first.kind()) => { self.items.push(item.clone(), styles); self.tight &= self.staged.drain(..).all(|(t, _)| *t != Content::Parbreak); @@ -751,21 +751,16 @@ impl<'a> ListBuilder<'a> { fn finish(self, parent: &mut Builder<'a>) -> SourceResult<()> { let (items, shared) = self.items.finish(); let kind = match items.items().next() { - Some(item) => item.kind, + Some(item) => item.kind(), None => return Ok(()), }; - let start = 1; let tight = self.tight; let attached = tight && self.attachable; - let content = match kind { - UNORDERED => { - Content::show(ListNode:: { start, tight, attached, items }) - } - ORDERED | _ => { - Content::show(ListNode:: { start, tight, attached, items }) - } + LIST => Content::show(ListNode:: { tight, attached, items }), + ENUM => Content::show(ListNode:: { tight, attached, items }), + DESC | _ => Content::show(ListNode:: { tight, attached, items }), }; let stored = parent.scratch.templates.alloc(content); diff --git a/src/model/property.rs b/src/model/property.rs index 18f41eee5..ab4f02e35 100644 --- a/src/model/property.rs +++ b/src/model/property.rs @@ -9,7 +9,7 @@ use super::{Interruption, NodeId, StyleChain}; use crate::eval::{RawLength, Smart}; use crate::geom::{Corners, Length, Numeric, Relative, Sides, Spec}; use crate::library::layout::PageNode; -use crate::library::structure::{EnumNode, ListNode}; +use crate::library::structure::{DescNode, EnumNode, ListNode}; use crate::library::text::ParNode; use crate::util::ReadableTypeId; @@ -68,7 +68,10 @@ impl Property { Some(Interruption::Page) } else if self.is_of::() { Some(Interruption::Par) - } else if self.is_of::() || self.is_of::() { + } else if self.is_of::() + || self.is_of::() + || self.is_of::() + { Some(Interruption::List) } else { None diff --git a/src/model/recipe.rs b/src/model/recipe.rs index 6b21ccf20..27b1be424 100644 --- a/src/model/recipe.rs +++ b/src/model/recipe.rs @@ -5,7 +5,7 @@ use comemo::Tracked; use super::{Content, Interruption, NodeId, Show, ShowNode, StyleChain, StyleEntry}; use crate::diag::SourceResult; use crate::eval::{Args, Func, Regex, Value}; -use crate::library::structure::{EnumNode, ListNode}; +use crate::library::structure::{DescNode, EnumNode, ListNode}; use crate::syntax::Spanned; use crate::World; @@ -93,7 +93,10 @@ impl Recipe { /// What kind of structure the property interrupts. pub fn interruption(&self) -> Option { if let Pattern::Node(id) = self.pattern { - if id == NodeId::of::() || id == NodeId::of::() { + if id == NodeId::of::() + || id == NodeId::of::() + || id == NodeId::of::() + { return Some(Interruption::List); } } diff --git a/src/parse/incremental.rs b/src/parse/incremental.rs index 9272dcfc4..06096a752 100644 --- a/src/parse/incremental.rs +++ b/src/parse/incremental.rs @@ -407,18 +407,18 @@ mod tests { test("", 0..0, "do it", 0..5); test("a d e", 1 .. 3, " b c d", 0 .. 9); test("*~ *", 2..2, "*", 0..5); - test("_1_\n2a\n3", 5..5, "4", 0..7); - test("_1_\n2a\n3~", 8..8, "4", 5..10); + test("_1_\n2a\n3", 5..5, "4", 4..7); + test("_1_\n2a\n3~", 8..8, "4", 4..10); test("_1_ 2 3a\n4", 7..7, "5", 0..9); test("* {1+2} *", 5..6, "3", 2..7); test("a #f() e", 1 .. 6, " b c d", 0 .. 9); test("a\nb\nc\nd\ne\n", 5 .. 5, "c", 2 .. 7); test("a\n\nb\n\nc\n\nd\n\ne\n", 7 .. 7, "c", 3 .. 10); - test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 6..20); + test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 4..20); test("~~ {a} ~~", 4 .. 5, "b", 3 .. 6); test("{(0, 1, 2)}", 5 .. 6, "11pt", 0..14); test("\n= A heading", 4 .. 4, "n evocative", 0 .. 23); - test("for~your~thing", 9 .. 9, "a", 4 .. 15); + test("for~your~thing", 9 .. 9, "a", 0 .. 15); test("a your thing a", 6 .. 7, "a", 0 .. 14); test("{call(); abc}", 7 .. 7, "[]", 0 .. 15); test("#call() abc", 7 .. 7, "[]", 0 .. 10); @@ -429,17 +429,17 @@ mod tests { test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 33 .. 40); test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 0 .. 33); test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 0 .. 33); - test("hello~~{x}", 7 .. 10, "#f()", 5 .. 11); - test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 5 .. 25); - test("understanding `code` is complicated", 15 .. 15, "C ", 14 .. 22); + test("hello~~{x}", 7 .. 10, "#f()", 0 .. 11); + test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 0 .. 25); + test("understanding `code` is complicated", 15 .. 15, "C ", 0 .. 22); test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17); - test(r#"a ```typst hello``` b"#, 16 .. 17, "", 2 .. 18); - test(r#"a ```typst hello```"#, 16 .. 17, "", 2 .. 18); + test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 18); + test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18); test("#for", 4 .. 4, "//", 0 .. 6); test("#show a: f as b..", 16..16, "c", 0..18); test("a\n#let \nb", 7 .. 7, "i", 2 .. 9); test("a\n#for i \nb", 9 .. 9, "in", 2 .. 12); - test("a~https://fun/html", 13..14, "n", 2..18); + test("a~https://fun/html", 13..14, "n", 0..18); } #[test] @@ -452,7 +452,7 @@ mod tests { test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19); test("#let x = (1, 2 + ;~ Five\r\n\r", 20 .. 23, "2.", 0 .. 23); test("hey #myfriend", 4 .. 4, "\\", 0 .. 14); - test("hey #myfriend", 4 .. 4, "\\", 3 .. 6); + test("hey #myfriend", 4 .. 4, "\\", 0 .. 6); test("= foo\nbar\n - a\n - b", 6 .. 9, "", 0 .. 11); test("= foo\n bar\n baz", 6 .. 8, "", 0 .. 9); test(" // hi", 1 .. 1, " ", 0 .. 7); @@ -461,12 +461,12 @@ mod tests { #[test] fn test_parse_incremental_type_invariants() { - test("a #for x in array {x}", 18 .. 21, "[#x]", 2 .. 22); - test("a #let x = 1 {5}", 3 .. 6, "if", 2 .. 11); + test("a #for x in array {x}", 18 .. 21, "[#x]", 0 .. 22); + test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 11); test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16); test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 13); test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18); - test("a // b c #f()", 3 .. 4, "", 2 .. 12); + test("a // b c #f()", 3 .. 4, "", 0 .. 12); test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12); test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13); test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26); @@ -477,8 +477,8 @@ mod tests { #[test] fn test_parse_incremental_wrongly_or_unclosed_things() { test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6); - test(r"this \u{abcd}", 8 .. 9, "", 5 .. 12); - test(r"this \u{abcd} that", 12 .. 13, "", 5 .. 17); + test(r"this \u{abcd}", 8 .. 9, "", 0 .. 12); + test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17); test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24); test("a b c", 1 .. 1, " /* letters */", 0 .. 19); test("a b c", 1 .. 1, " /* letters", 0 .. 16); diff --git a/src/parse/mod.rs b/src/parse/mod.rs index a8dee7f38..ed8bc5ced 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -162,11 +162,6 @@ fn markup(p: &mut Parser, mut at_start: bool) { }); } -/// Parse a single line of markup. -fn markup_line(p: &mut Parser) { - markup_indented(p, usize::MAX); -} - /// Parse markup that stays right of the given `column`. fn markup_indented(p: &mut Parser, min_indent: usize) { p.eat_while(|t| match t { @@ -185,7 +180,6 @@ fn markup_indented(p: &mut Parser, min_indent: usize) { { break; } - Some(NodeKind::Label(_)) => break, _ => {} } @@ -195,6 +189,33 @@ fn markup_indented(p: &mut Parser, min_indent: usize) { marker.end(p, NodeKind::Markup { min_indent }); } +/// Parse a line of markup that can prematurely end if `f` returns true. +fn markup_line(p: &mut Parser, mut f: F) +where + F: FnMut(&NodeKind) -> bool, +{ + p.eat_while(|t| match t { + NodeKind::Space { newlines } => *newlines == 0, + NodeKind::LineComment | NodeKind::BlockComment => true, + _ => false, + }); + + p.perform(NodeKind::Markup { min_indent: usize::MAX }, |p| { + let mut at_start = false; + while let Some(kind) = p.peek() { + if let NodeKind::Space { newlines: (1 ..) } = kind { + break; + } + + if f(kind) { + break; + } + + markup_node(p, &mut at_start); + } + }); +} + /// Parse a markup node. fn markup_node(p: &mut Parser, at_start: &mut bool) { let token = match p.peek() { @@ -226,6 +247,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { | NodeKind::Ellipsis | NodeKind::Quote { .. } | NodeKind::Escape(_) + | NodeKind::Link(_) | NodeKind::Raw(_) | NodeKind::Math(_) | NodeKind::Label(_) @@ -233,12 +255,22 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) { p.eat(); } - // Grouping markup. + // Strong, emph, heading. NodeKind::Star => strong(p), NodeKind::Underscore => emph(p), NodeKind::Eq => heading(p, *at_start), + + // Lists. NodeKind::Minus => list_node(p, *at_start), - NodeKind::EnumNumbering(_) => enum_node(p, *at_start), + NodeKind::Plus | NodeKind::EnumNumbering(_) => enum_node(p, *at_start), + NodeKind::Slash => { + desc_node(p, *at_start).ok(); + } + NodeKind::Colon => { + let marker = p.marker(); + p.eat(); + marker.convert(p, NodeKind::Text(':'.into())); + } // Hashtag + keyword / identifier. NodeKind::Ident(_) @@ -293,7 +325,7 @@ fn heading(p: &mut Parser, at_start: bool) { if at_start && p.peek().map_or(true, |kind| kind.is_space()) { p.eat_while(|kind| *kind == NodeKind::Space { newlines: 0 }); - markup_line(p); + markup_line(p, |kind| matches!(kind, NodeKind::Label(_))); marker.end(p, NodeKind::Heading); } else { let text = p.get(current_start .. p.prev_end()).into(); @@ -331,6 +363,25 @@ fn enum_node(p: &mut Parser, at_start: bool) { } } +/// Parse a single description list item. +fn desc_node(p: &mut Parser, at_start: bool) -> ParseResult { + let marker = p.marker(); + let text: EcoString = p.peek_src().into(); + p.eat(); + + let min_indent = p.column(p.prev_end()); + if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() { + markup_line(p, |node| matches!(node, NodeKind::Colon)); + p.expect(NodeKind::Colon)?; + markup_indented(p, min_indent); + marker.end(p, NodeKind::Desc); + } else { + marker.convert(p, NodeKind::Text(text)); + } + + Ok(()) +} + /// Parse an expression within a markup mode. fn markup_expr(p: &mut Parser) { // Does the expression need termination or can content follow directly? diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs index bce2da3cc..d68282c0a 100644 --- a/src/parse/resolve.rs +++ b/src/parse/resolve.rs @@ -26,14 +26,12 @@ pub fn resolve_string(string: &str) -> EcoString { // TODO: Error if closing brace is missing. let sequence = s.eat_while(char::is_ascii_hexdigit); let _terminated = s.eat_if('}'); - match resolve_hex(sequence) { Some(c) => out.push(c), None => out.push_str(s.from(start)), } } - // TODO: Error for invalid escape sequence. _ => out.push_str(s.from(start)), } } diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 84a63ba13..f6d4b0e86 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -103,6 +103,11 @@ impl<'s> Iterator for Tokens<'s> { let start = self.s.cursor(); let c = self.s.eat()?; Some(match c { + // Comments. + '/' if self.s.eat_if('/') => self.line_comment(), + '/' if self.s.eat_if('*') => self.block_comment(), + '*' if self.s.eat_if('/') => NodeKind::Unknown("*/".into()), + // Blocks. '{' => NodeKind::LeftBrace, '}' => NodeKind::RightBrace, @@ -110,15 +115,7 @@ impl<'s> Iterator for Tokens<'s> { ']' => NodeKind::RightBracket, // Whitespace. - ' ' if self.s.done() || !self.s.at(char::is_whitespace) => { - NodeKind::Space { newlines: 0 } - } - c if c.is_whitespace() => self.whitespace(), - - // Comments with special case for URLs. - '/' if self.s.eat_if('*') => self.block_comment(), - '/' if !self.maybe_in_url() && self.s.eat_if('/') => self.line_comment(), - '*' if self.s.eat_if('/') => NodeKind::Unknown(self.s.from(start).into()), + c if c.is_whitespace() => self.whitespace(c), // Other things. _ => match self.mode { @@ -130,122 +127,50 @@ impl<'s> Iterator for Tokens<'s> { } impl<'s> Tokens<'s> { - #[inline] - fn markup(&mut self, start: usize, c: char) -> NodeKind { - match c { - // Escape sequences. - '\\' => self.backslash(), - - // Keywords and identifiers. - '#' => self.hash(), - - // Markup. - '~' => NodeKind::NonBreakingSpace, - '-' => self.hyph(), - '.' if self.s.eat_if("..") => NodeKind::Ellipsis, - '\'' => NodeKind::Quote { double: false }, - '"' => NodeKind::Quote { double: true }, - '*' if !self.in_word() => NodeKind::Star, - '_' if !self.in_word() => NodeKind::Underscore, - '`' => self.raw(), - '=' => NodeKind::Eq, - '$' => self.math(), - '<' => self.label(), - '@' => self.reference(), - c if c == '.' || c.is_ascii_digit() => self.numbering(start, c), - - // Plain text. - _ => self.text(start), + fn line_comment(&mut self) -> NodeKind { + self.s.eat_until(is_newline); + if self.s.peek().is_none() { + self.terminated = false; } + NodeKind::LineComment } - fn code(&mut self, start: usize, c: char) -> NodeKind { - match c { - // Parens. - '(' => NodeKind::LeftParen, - ')' => NodeKind::RightParen, + fn block_comment(&mut self) -> NodeKind { + let mut state = '_'; + let mut depth = 1; + self.terminated = false; - // Length two. - '=' if self.s.eat_if('=') => NodeKind::EqEq, - '!' if self.s.eat_if('=') => NodeKind::ExclEq, - '<' if self.s.eat_if('=') => NodeKind::LtEq, - '>' if self.s.eat_if('=') => NodeKind::GtEq, - '+' if self.s.eat_if('=') => NodeKind::PlusEq, - '-' if self.s.eat_if('=') => NodeKind::HyphEq, - '*' if self.s.eat_if('=') => NodeKind::StarEq, - '/' if self.s.eat_if('=') => NodeKind::SlashEq, - '.' if self.s.eat_if('.') => NodeKind::Dots, - '=' if self.s.eat_if('>') => NodeKind::Arrow, - - // Length one. - ',' => NodeKind::Comma, - ';' => NodeKind::Semicolon, - ':' => NodeKind::Colon, - '+' => NodeKind::Plus, - '-' => NodeKind::Minus, - '*' => NodeKind::Star, - '/' => NodeKind::Slash, - '=' => NodeKind::Eq, - '<' => NodeKind::Lt, - '>' => NodeKind::Gt, - '.' if self.s.done() || !self.s.at(char::is_ascii_digit) => NodeKind::Dot, - - // Identifiers. - c if is_id_start(c) => self.ident(start), - - // Numbers. - c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => { - self.number(start, c) + // Find the first `*/` that does not correspond to a nested `/*`. + while let Some(c) = self.s.eat() { + state = match (state, c) { + ('*', '/') => { + depth -= 1; + if depth == 0 { + self.terminated = true; + break; + } + '_' + } + ('/', '*') => { + depth += 1; + '_' + } + ('/', '/') => { + self.line_comment(); + '_' + } + _ => c, } - - // Strings. - '"' => self.string(), - - _ => NodeKind::Unknown(self.s.from(start).into()), } + + NodeKind::BlockComment } - #[inline] - fn text(&mut self, start: usize) -> NodeKind { - macro_rules! table { - ($($c:literal)|*) => {{ - let mut t = [false; 128]; - $(t[$c as usize] = true;)* - t - }} + fn whitespace(&mut self, c: char) -> NodeKind { + if c == ' ' && !self.s.at(char::is_whitespace) { + return NodeKind::Space { newlines: 0 }; } - const TABLE: [bool; 128] = table! { - // Ascii whitespace. - ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | - // Comments, parentheses, code. - '/' | '[' | ']' | '{' | '}' | '#' | - // Markup - '~' | '-' | '.' | '\'' | '"' | '*' | '_' | '`' | '$' | '\\' - }; - - loop { - self.s.eat_until(|c: char| { - TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace()) - }); - - // Allow a single space, optionally preceded by . or - if something - // alphanumeric follows directly. This leads to less text nodes, - // which is good for performance. - let mut s = self.s; - s.eat_if(['.', '-']); - s.eat_if(' '); - if !s.at(char::is_alphanumeric) { - break; - } - - self.s = s; - } - - NodeKind::Text(self.s.from(start).into()) - } - - fn whitespace(&mut self) -> NodeKind { self.s.uneat(); // Count the number of newlines. @@ -267,25 +192,84 @@ impl<'s> Tokens<'s> { NodeKind::Space { newlines } } - fn backslash(&mut self) -> NodeKind { - let c = match self.s.peek() { - Some(c) => c, - None => return NodeKind::Linebreak { justified: false }, + #[inline] + fn markup(&mut self, start: usize, c: char) -> NodeKind { + match c { + // Escape sequences. + '\\' => self.backslash(), + + // Single-char things. + '~' => NodeKind::NonBreakingSpace, + '.' if self.s.eat_if("..") => NodeKind::Ellipsis, + '\'' => NodeKind::Quote { double: false }, + '"' => NodeKind::Quote { double: true }, + '*' if !self.in_word() => NodeKind::Star, + '_' if !self.in_word() => NodeKind::Underscore, + '=' => NodeKind::Eq, + '+' => NodeKind::Plus, + '/' => NodeKind::Slash, + ':' => NodeKind::Colon, + + // Multi-char things. + '#' => self.hash(start), + '-' => self.hyph(), + 'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => { + self.link(start) + } + '`' => self.raw(), + '$' => self.math(), + c if c.is_ascii_digit() => self.numbering(start), + '<' => self.label(), + '@' => self.reference(start), + + // Plain text. + _ => self.text(start), + } + } + + #[inline] + fn text(&mut self, start: usize) -> NodeKind { + macro_rules! table { + ($(|$c:literal)*) => {{ + let mut t = [false; 128]; + $(t[$c as usize] = true;)* + t + }} + } + + const TABLE: [bool; 128] = table! { + | ' ' | '\t' | '\n' | '\x0b' | '\x0c' | '\r' | '\\' | '/' + | '[' | ']' | '{' | '}' | '~' | '-' | '.' | '\'' | '"' + | '*' | '_' | ':' | 'h' | '`' | '$' | '<' | '>' | '@' | '#' }; - match c { - // Backslash and comments. - '\\' | '/' | - // Parenthesis and hashtag. - '[' | ']' | '{' | '}' | '#' | - // Markup. - '~' | '-' | '.' | ':' | - '\'' | '"' | '*' | '_' | '`' | '$' | '=' | - '<' | '>' | '@' => { - self.s.expect(c); - NodeKind::Escape(c) + loop { + self.s.eat_until(|c: char| { + TABLE.get(c as usize).copied().unwrap_or_else(|| c.is_whitespace()) + }); + + // Continue with the same text node if the thing would become text + // anyway. + let mut s = self.s; + match s.eat() { + Some('/') if !s.at(['/', '*']) => {} + Some(' ') if s.at(char::is_alphanumeric) => {} + Some('-') if !s.at(['-', '?']) => {} + Some('.') if !s.at("..") => {} + Some('h') if !s.at("ttp://") && !s.at("ttps://") => {} + Some('@' | '#') if !s.at(is_id_start) => {} + _ => break, } - 'u' if self.s.eat_if("u{") => { + + self.s = s; + } + + NodeKind::Text(self.s.from(start).into()) + } + + fn backslash(&mut self) -> NodeKind { + match self.s.peek() { + Some('u') if self.s.eat_if("u{") => { let sequence = self.s.eat_while(char::is_ascii_alphanumeric); if self.s.eat_if('}') { if let Some(c) = resolve_hex(sequence) { @@ -298,26 +282,23 @@ impl<'s> Tokens<'s> { } } else { self.terminated = false; - NodeKind::Error( - SpanPos::End, - "expected closing brace".into(), - ) + NodeKind::Error(SpanPos::End, "expected closing brace".into()) } } // Linebreaks. - c if c.is_whitespace() => NodeKind::Linebreak { justified: false }, - '+' => { - self.s.expect(c); - NodeKind::Linebreak { justified: true } - } + Some(c) if c.is_whitespace() => NodeKind::Linebreak, + None => NodeKind::Linebreak, - // Just the backslash. - _ => NodeKind::Text('\\'.into()), + // Escapes. + Some(c) => { + self.s.expect(c); + NodeKind::Escape(c) + } } } - fn hash(&mut self) -> NodeKind { + fn hash(&mut self, start: usize) -> NodeKind { if self.s.at(is_id_start) { let read = self.s.eat_while(is_id_continue); match keyword(read) { @@ -325,7 +306,7 @@ impl<'s> Tokens<'s> { None => NodeKind::Ident(read.into()), } } else { - NodeKind::Text('#'.into()) + self.text(start) } } @@ -343,19 +324,26 @@ impl<'s> Tokens<'s> { } } - fn numbering(&mut self, start: usize, c: char) -> NodeKind { - let number = if c != '.' { - self.s.eat_while(char::is_ascii_digit); - let read = self.s.from(start); - if !self.s.eat_if('.') { - return NodeKind::Text(self.s.from(start).into()); - } - read.parse().ok() - } else { - None - }; + fn in_word(&self) -> bool { + let alphanumeric = |c: Option| c.map_or(false, |c| c.is_alphanumeric()); + let prev = self.s.scout(-2); + let next = self.s.peek(); + alphanumeric(prev) && alphanumeric(next) + } - NodeKind::EnumNumbering(number) + fn link(&mut self, start: usize) -> NodeKind { + #[rustfmt::skip] + self.s.eat_while(|c: char| matches!(c, + | '0' ..= '9' + | 'a' ..= 'z' + | 'A' ..= 'Z' + | '~' | '/' | '%' | '?' | '#' | '&' | '+' | '=' + | '\'' | '.' | ',' | ';' + )); + if self.s.scout(-1) == Some('.') { + self.s.uneat(); + } + NodeKind::Link(self.s.from(start).into()) } fn raw(&mut self) -> NodeKind { @@ -376,7 +364,6 @@ impl<'s> Tokens<'s> { } let start = self.s.cursor(); - let mut found = 0; while found < backticks { match self.s.eat() { @@ -394,10 +381,9 @@ impl<'s> Tokens<'s> { self.s.get(start .. end), ))) } else { + self.terminated = false; let remaining = backticks - found; let noun = if remaining == 1 { "backtick" } else { "backticks" }; - - self.terminated = false; NodeKind::Error( SpanPos::End, if found == 0 { @@ -410,53 +396,40 @@ impl<'s> Tokens<'s> { } fn math(&mut self) -> NodeKind { - let mut display = false; - if self.s.eat_if('[') { - display = true; - } - - let start = self.s.cursor(); - let mut escaped = false; - let mut dollar = !display; - - let terminated = loop { - match self.s.eat() { - Some('$') if !escaped && dollar => break true, - Some(']') if !escaped => dollar = true, - Some(c) => { - dollar = !display; - escaped = c == '\\' && !escaped; - } - None => break false, + let formula = self.s.eat_until(|c| { + if c == '$' && !escaped { + true + } else { + escaped = c == '\\' && !escaped; + false } - }; + }); - let end = self.s.cursor() - - match (terminated, display) { - (false, _) => 0, - (true, false) => 1, - (true, true) => 2, - }; + let display = formula.len() >= 2 + && formula.starts_with(char::is_whitespace) + && formula.ends_with(char::is_whitespace); - if terminated { - NodeKind::Math(Arc::new(MathNode { - formula: self.s.get(start .. end).into(), - display, - })) + if self.s.eat_if('$') { + NodeKind::Math(Arc::new(MathNode { formula: formula.into(), display })) } else { self.terminated = false; - NodeKind::Error( - SpanPos::End, - if !display || (!escaped && dollar) { - "expected closing dollar sign".into() - } else { - "expected closing bracket and dollar sign".into() - }, - ) + NodeKind::Error(SpanPos::End, "expected dollar sign".into()) } } + fn numbering(&mut self, start: usize) -> NodeKind { + self.s.eat_while(char::is_ascii_digit); + let read = self.s.from(start); + if self.s.eat_if('.') { + if let Ok(number) = read.parse() { + return NodeKind::EnumNumbering(number); + } + } + + self.text(start) + } + fn label(&mut self) -> NodeKind { let label = self.s.eat_while(is_id_continue); if self.s.eat_if('>') { @@ -471,12 +444,59 @@ impl<'s> Tokens<'s> { } } - fn reference(&mut self) -> NodeKind { + fn reference(&mut self, start: usize) -> NodeKind { let label = self.s.eat_while(is_id_continue); if !label.is_empty() { NodeKind::Ref(label.into()) } else { - NodeKind::Error(SpanPos::Full, "label cannot be empty".into()) + self.text(start) + } + } + + fn code(&mut self, start: usize, c: char) -> NodeKind { + match c { + // Parentheses. + '(' => NodeKind::LeftParen, + ')' => NodeKind::RightParen, + + // Two-char operators. + '=' if self.s.eat_if('=') => NodeKind::EqEq, + '!' if self.s.eat_if('=') => NodeKind::ExclEq, + '<' if self.s.eat_if('=') => NodeKind::LtEq, + '>' if self.s.eat_if('=') => NodeKind::GtEq, + '+' if self.s.eat_if('=') => NodeKind::PlusEq, + '-' if self.s.eat_if('=') => NodeKind::HyphEq, + '*' if self.s.eat_if('=') => NodeKind::StarEq, + '/' if self.s.eat_if('=') => NodeKind::SlashEq, + '.' if self.s.eat_if('.') => NodeKind::Dots, + '=' if self.s.eat_if('>') => NodeKind::Arrow, + + // Single-char operators. + ',' => NodeKind::Comma, + ';' => NodeKind::Semicolon, + ':' => NodeKind::Colon, + '+' => NodeKind::Plus, + '-' => NodeKind::Minus, + '*' => NodeKind::Star, + '/' => NodeKind::Slash, + '=' => NodeKind::Eq, + '<' => NodeKind::Lt, + '>' => NodeKind::Gt, + '.' if !self.s.at(char::is_ascii_digit) => NodeKind::Dot, + + // Identifiers. + c if is_id_start(c) => self.ident(start), + + // Numbers. + c if c.is_ascii_digit() || (c == '.' && self.s.at(char::is_ascii_digit)) => { + self.number(start, c) + } + + // Strings. + '"' => self.string(), + + // Invalid token. + _ => NodeKind::Unknown(self.s.from(start).into()), } } @@ -543,18 +563,18 @@ impl<'s> Tokens<'s> { } } - fn string(&mut self) -> NodeKind { let mut escaped = false; - let string = resolve_string(self.s.eat_until(|c| { + let verbatim = self.s.eat_until(|c| { if c == '"' && !escaped { true } else { escaped = c == '\\' && !escaped; false } - })); + }); + let string = resolve_string(verbatim); if self.s.eat_if('"') { NodeKind::Str(string) } else { @@ -562,56 +582,6 @@ impl<'s> Tokens<'s> { NodeKind::Error(SpanPos::End, "expected quote".into()) } } - - fn line_comment(&mut self) -> NodeKind { - self.s.eat_until(is_newline); - if self.s.peek().is_none() { - self.terminated = false; - } - NodeKind::LineComment - } - - fn block_comment(&mut self) -> NodeKind { - let mut state = '_'; - let mut depth = 1; - self.terminated = false; - - // Find the first `*/` that does not correspond to a nested `/*`. - while let Some(c) = self.s.eat() { - state = match (state, c) { - ('*', '/') => { - depth -= 1; - if depth == 0 { - self.terminated = true; - break; - } - '_' - } - ('/', '*') => { - depth += 1; - '_' - } - ('/', '/') => { - self.line_comment(); - '_' - } - _ => c, - } - } - - NodeKind::BlockComment - } - - fn in_word(&self) -> bool { - let alphanumeric = |c: Option| c.map_or(false, |c| c.is_alphanumeric()); - let prev = self.s.scout(-2); - let next = self.s.peek(); - alphanumeric(prev) && alphanumeric(next) - } - - fn maybe_in_url(&self) -> bool { - self.mode == TokenMode::Markup && self.s.before().ends_with(":/") - } } fn keyword(ident: &str) -> Option { @@ -872,14 +842,14 @@ mod tests { #[test] fn test_tokenize_text() { // Test basic text. - t!(Markup[" /"]: "hello" => Text("hello")); - t!(Markup[" /"]: "hello-world" => Text("hello-world")); + t!(Markup[" /"]: "hello" => Text("hello")); + t!(Markup[" /"]: "reha-world" => Text("reha-world")); // Test code symbols in text. - t!(Markup[" /"]: "a():\"b" => Text("a():"), Quote { double: true }, Text("b")); - t!(Markup[" /"]: ";:,|/+" => Text(";:,|"), Text("/+")); + t!(Markup[" /"]: "a():\"b" => Text("a()"), Colon, Quote { double: true }, Text("b")); + t!(Markup[" /"]: ";,|/+" => Text(";,|/+")); t!(Markup[" /"]: "=-a" => Eq, Minus, Text("a")); - t!(Markup[" "]: "#123" => Text("#"), Text("123")); + t!(Markup[" "]: "#123" => Text("#123")); // Test text ends. t!(Markup[""]: "hello " => Text("hello"), Space(0)); @@ -904,11 +874,9 @@ mod tests { t!(Markup: r"\`" => Escape('`')); t!(Markup: r"\$" => Escape('$')); t!(Markup: r"\#" => Escape('#')); - - // Test unescapable symbols. - t!(Markup[" /"]: r"\a" => Text(r"\"), Text("a")); - t!(Markup[" /"]: r"\u" => Text(r"\"), Text("u")); - t!(Markup[" /"]: r"\1" => Text(r"\"), Text("1")); + t!(Markup: r"\a" => Escape('a')); + t!(Markup: r"\u" => Escape('u')); + t!(Markup: r"\1" => Escape('1')); // Test basic unicode escapes. t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence")); @@ -930,16 +898,15 @@ mod tests { t!(Markup: "_" => Underscore); t!(Markup[""]: "===" => Eq, Eq, Eq); t!(Markup["a1/"]: "= " => Eq, Space(0)); - t!(Markup[" "]: r"\" => Linebreak { justified: false }); - t!(Markup[" "]: r"\+" => Linebreak { justified: true }); + t!(Markup[" "]: r"\" => Linebreak); t!(Markup: "~" => NonBreakingSpace); t!(Markup["a1/"]: "-?" => Shy); t!(Markup["a "]: r"a--" => Text("a"), EnDash); t!(Markup["a1/"]: "- " => Minus, Space(0)); - t!(Markup[" "]: "." => EnumNumbering(None)); - t!(Markup[" "]: "1." => EnumNumbering(Some(1))); - t!(Markup[" "]: "1.a" => EnumNumbering(Some(1)), Text("a")); - t!(Markup[" /"]: "a1." => Text("a1"), EnumNumbering(None)); + t!(Markup[" "]: "+" => Plus); + t!(Markup[" "]: "1." => EnumNumbering(1)); + t!(Markup[" "]: "1.a" => EnumNumbering(1), Text("a")); + t!(Markup[" /"]: "a1." => Text("a1.")); } #[test] @@ -995,7 +962,7 @@ mod tests { for (s, t) in list.clone() { t!(Markup[" "]: format!("#{}", s) => t); t!(Markup[" "]: format!("#{0}#{0}", s) => t, t); - t!(Markup[" /"]: format!("# {}", s) => Text("#"), Space(0), Text(s)); + t!(Markup[" /"]: format!("# {}", s) => Text(&format!("# {s}"))); } for (s, t) in list { @@ -1037,18 +1004,16 @@ mod tests { t!(Markup: "$$" => Math("", false)); t!(Markup: "$x$" => Math("x", false)); t!(Markup: r"$\\$" => Math(r"\\", false)); - t!(Markup: "$[x + y]$" => Math("x + y", true)); - t!(Markup: r"$[\\]$" => Math(r"\\", true)); + t!(Markup: r"$[\\]$" => Math(r"[\\]", false)); + t!(Markup: "$ x + y $" => Math(" x + y ", true)); // Test unterminated. - t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign")); - t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign")); - t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign")); + t!(Markup[""]: "$x" => Error(End, "expected dollar sign")); + t!(Markup[""]: "$[x]\n" => Error(End, "expected dollar sign")); // Test escape sequences. - t!(Markup: r"$\$x$" => Math(r"\$x", false)); - t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true)); - t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign")); + t!(Markup: r"$\$x$" => Math(r"\$x", false)); + t!(Markup: r"$\ \$ $" => Math(r"\ \$ ", false)); } #[test] diff --git a/src/syntax/ast.rs b/src/syntax/ast.rs index 10bee4e82..8d3696a89 100644 --- a/src/syntax/ast.rs +++ b/src/syntax/ast.rs @@ -63,9 +63,7 @@ impl Markup { self.0.children().filter_map(|node| match node.kind() { NodeKind::Space { newlines: (2 ..) } => Some(MarkupNode::Parbreak), NodeKind::Space { .. } => Some(MarkupNode::Space), - &NodeKind::Linebreak { justified } => { - Some(MarkupNode::Linebreak { justified }) - } + NodeKind::Linebreak => Some(MarkupNode::Linebreak), NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())), NodeKind::Escape(c) => Some(MarkupNode::Text((*c).into())), NodeKind::NonBreakingSpace => Some(MarkupNode::Text('\u{00A0}'.into())), @@ -76,6 +74,7 @@ impl Markup { &NodeKind::Quote { double } => Some(MarkupNode::Quote { double }), NodeKind::Strong => node.cast().map(MarkupNode::Strong), NodeKind::Emph => node.cast().map(MarkupNode::Emph), + NodeKind::Link(url) => Some(MarkupNode::Link(url.clone())), NodeKind::Raw(raw) => Some(MarkupNode::Raw(raw.as_ref().clone())), NodeKind::Math(math) => Some(MarkupNode::Math(Spanned::new( math.as_ref().clone(), @@ -84,6 +83,7 @@ impl Markup { NodeKind::Heading => node.cast().map(MarkupNode::Heading), NodeKind::List => node.cast().map(MarkupNode::List), NodeKind::Enum => node.cast().map(MarkupNode::Enum), + NodeKind::Desc => node.cast().map(MarkupNode::Desc), NodeKind::Label(v) => Some(MarkupNode::Label(v.clone())), NodeKind::Ref(v) => Some(MarkupNode::Ref(v.clone())), _ => node.cast().map(MarkupNode::Expr), @@ -96,8 +96,8 @@ impl Markup { pub enum MarkupNode { /// Whitespace containing less than two newlines. Space, - /// A forced line break: `\` or `\+` if justified. - Linebreak { justified: bool }, + /// A forced line break. + Linebreak, /// A paragraph break: Two or more newlines. Parbreak, /// Plain text. @@ -108,6 +108,8 @@ pub enum MarkupNode { Strong(StrongNode), /// Emphasized content: `_Emphasized_`. Emph(EmphNode), + /// A hyperlink. + Link(EcoString), /// A raw block with optional syntax highlighting: `` `...` ``. Raw(RawNode), /// A math formula: `$a^2 = b^2 + c^2$`. @@ -116,8 +118,10 @@ pub enum MarkupNode { Heading(HeadingNode), /// An item in an unordered list: `- ...`. List(ListNode), - /// An item in an enumeration (ordered list): `1. ...`. + /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. Enum(EnumNode), + /// An item in a description list: `/ Term: Details. + Desc(DescNode), /// A label. Label(EcoString), /// A reference. @@ -170,8 +174,8 @@ pub struct RawNode { pub struct MathNode { /// The formula between the dollars / brackets. pub formula: EcoString, - /// Whether the formula is display-level, that is, it is surrounded by - /// `$[..]$`. + /// Whether the formula is display-level, that is, it contains whitespace + /// after the starting dollar sign and before the ending dollar sign. pub display: bool, } @@ -205,7 +209,7 @@ node! { impl ListNode { /// The contents of the list item. pub fn body(&self) -> Markup { - self.0.cast_first_child().expect("list node is missing body") + self.0.cast_first_child().expect("list item is missing body") } } @@ -217,18 +221,36 @@ node! { impl EnumNode { /// The contents of the list item. pub fn body(&self) -> Markup { - self.0.cast_first_child().expect("enum node is missing body") + self.0.cast_first_child().expect("enum item is missing body") } /// The number, if any. pub fn number(&self) -> Option { + self.0.children().find_map(|node| match node.kind() { + NodeKind::EnumNumbering(num) => Some(*num), + _ => None, + }) + } +} + +node! { + /// An item in a description list: `/ Term: Details. + DescNode: Desc +} + +impl DescNode { + /// The term described by the list item. + pub fn term(&self) -> Markup { self.0 - .children() - .find_map(|node| match node.kind() { - NodeKind::EnumNumbering(num) => Some(*num), - _ => None, - }) - .expect("enum node is missing number") + .cast_first_child() + .expect("description list item is missing term") + } + + /// The description of the term. + pub fn body(&self) -> Markup { + self.0 + .cast_last_child() + .expect("description list item is missing body") } } diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs index 7f5ee0834..de7c70a2f 100644 --- a/src/syntax/highlight.rs +++ b/src/syntax/highlight.rs @@ -147,12 +147,12 @@ pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String { /// The syntax highlighting category of a node. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum Category { + /// A line or block comment. + Comment, /// Any kind of bracket, parenthesis or brace. Bracket, /// Punctuation in code. Punctuation, - /// A line or block comment. - Comment, /// An easily typable shortcut to a unicode codepoint. Shortcut, /// An escape sequence. @@ -161,14 +161,18 @@ pub enum Category { Strong, /// Emphasized text. Emph, + /// A hyperlink. + Link, /// Raw text or code. Raw, /// A math formula. Math, /// A section heading. Heading, - /// A list or enumeration. + /// A symbol of a list, enumeration, or description list. List, + /// A term in a description list. + Term, /// A label. Label, /// A reference. @@ -204,66 +208,74 @@ impl Category { i: usize, ) -> Option { match child.kind() { + NodeKind::LineComment => Some(Category::Comment), + NodeKind::BlockComment => Some(Category::Comment), NodeKind::LeftBrace => Some(Category::Bracket), NodeKind::RightBrace => Some(Category::Bracket), NodeKind::LeftBracket => Some(Category::Bracket), NodeKind::RightBracket => Some(Category::Bracket), NodeKind::LeftParen => Some(Category::Bracket), NodeKind::RightParen => Some(Category::Bracket), - NodeKind::Comma => Some(Category::Punctuation), - NodeKind::Semicolon => Some(Category::Punctuation), - NodeKind::Colon => Some(Category::Punctuation), - NodeKind::Dot => Some(Category::Punctuation), - NodeKind::LineComment => Some(Category::Comment), - NodeKind::BlockComment => Some(Category::Comment), + + NodeKind::Markup { .. } => match parent.kind() { + NodeKind::Desc + if parent + .children() + .take_while(|child| child.kind() != &NodeKind::Colon) + .find(|c| matches!(c.kind(), NodeKind::Markup { .. })) + .map_or(false, |ident| std::ptr::eq(ident, child)) => + { + Some(Category::Term) + } + _ => None, + }, + NodeKind::Space { .. } => None, NodeKind::Linebreak { .. } => Some(Category::Shortcut), + NodeKind::Text(_) => None, + NodeKind::Escape(_) => Some(Category::Escape), NodeKind::NonBreakingSpace => Some(Category::Shortcut), NodeKind::Shy => Some(Category::Shortcut), NodeKind::EnDash => Some(Category::Shortcut), NodeKind::EmDash => Some(Category::Shortcut), NodeKind::Ellipsis => Some(Category::Shortcut), - NodeKind::Escape(_) => Some(Category::Escape), - NodeKind::Strong => Some(Category::Strong), - NodeKind::Emph => Some(Category::Emph), - NodeKind::Raw(_) => Some(Category::Raw), - NodeKind::Math(_) => Some(Category::Math), - NodeKind::Heading => Some(Category::Heading), - NodeKind::Minus => match parent.kind() { - NodeKind::List => Some(Category::List), - _ => Some(Category::Operator), - }, - NodeKind::EnumNumbering(_) => Some(Category::List), - NodeKind::Label(_) => Some(Category::Label), - NodeKind::Ref(_) => Some(Category::Ref), - NodeKind::Not => Some(Category::Keyword), - NodeKind::And => Some(Category::Keyword), - NodeKind::Or => Some(Category::Keyword), - NodeKind::Let => Some(Category::Keyword), - NodeKind::Set => Some(Category::Keyword), - NodeKind::Show => Some(Category::Keyword), - NodeKind::Wrap => Some(Category::Keyword), - NodeKind::If => Some(Category::Keyword), - NodeKind::Else => Some(Category::Keyword), - NodeKind::While => Some(Category::Keyword), - NodeKind::For => Some(Category::Keyword), - NodeKind::In => Some(Category::Keyword), - NodeKind::As => Some(Category::Keyword), - NodeKind::Break => Some(Category::Keyword), - NodeKind::Continue => Some(Category::Keyword), - NodeKind::Return => Some(Category::Keyword), - NodeKind::Import => Some(Category::Keyword), - NodeKind::From => Some(Category::Keyword), - NodeKind::Include => Some(Category::Keyword), - NodeKind::Plus => Some(Category::Operator), + NodeKind::Quote { .. } => None, NodeKind::Star => match parent.kind() { NodeKind::Strong => None, _ => Some(Category::Operator), }, - NodeKind::Slash => Some(Category::Operator), - NodeKind::PlusEq => Some(Category::Operator), - NodeKind::HyphEq => Some(Category::Operator), - NodeKind::StarEq => Some(Category::Operator), - NodeKind::SlashEq => Some(Category::Operator), + NodeKind::Underscore => None, + NodeKind::Strong => Some(Category::Strong), + NodeKind::Emph => Some(Category::Emph), + NodeKind::Link(_) => Some(Category::Link), + NodeKind::Raw(_) => Some(Category::Raw), + NodeKind::Math(_) => Some(Category::Math), + NodeKind::Heading => Some(Category::Heading), + NodeKind::List => None, + NodeKind::Enum => None, + NodeKind::EnumNumbering(_) => Some(Category::List), + NodeKind::Desc => None, + NodeKind::Label(_) => Some(Category::Label), + NodeKind::Ref(_) => Some(Category::Ref), + + NodeKind::Comma => Some(Category::Punctuation), + NodeKind::Semicolon => Some(Category::Punctuation), + NodeKind::Colon => match parent.kind() { + NodeKind::Desc => Some(Category::Term), + _ => Some(Category::Punctuation), + }, + NodeKind::Plus => match parent.kind() { + NodeKind::Enum => Some(Category::List), + _ => Some(Category::Operator), + }, + NodeKind::Minus => match parent.kind() { + NodeKind::List => Some(Category::List), + _ => Some(Category::Operator), + }, + NodeKind::Slash => match parent.kind() { + NodeKind::Desc => Some(Category::List), + _ => Some(Category::Operator), + }, + NodeKind::Dot => Some(Category::Punctuation), NodeKind::Eq => match parent.kind() { NodeKind::Heading => None, _ => Some(Category::Operator), @@ -274,10 +286,34 @@ impl Category { NodeKind::LtEq => Some(Category::Operator), NodeKind::Gt => Some(Category::Operator), NodeKind::GtEq => Some(Category::Operator), + NodeKind::PlusEq => Some(Category::Operator), + NodeKind::HyphEq => Some(Category::Operator), + NodeKind::StarEq => Some(Category::Operator), + NodeKind::SlashEq => Some(Category::Operator), NodeKind::Dots => Some(Category::Operator), NodeKind::Arrow => Some(Category::Operator), + NodeKind::Not => Some(Category::Keyword), + NodeKind::And => Some(Category::Keyword), + NodeKind::Or => Some(Category::Keyword), NodeKind::None => Some(Category::None), NodeKind::Auto => Some(Category::Auto), + NodeKind::Let => Some(Category::Keyword), + NodeKind::Set => Some(Category::Keyword), + NodeKind::Show => Some(Category::Keyword), + NodeKind::Wrap => Some(Category::Keyword), + NodeKind::If => Some(Category::Keyword), + NodeKind::Else => Some(Category::Keyword), + NodeKind::For => Some(Category::Keyword), + NodeKind::In => Some(Category::Keyword), + NodeKind::While => Some(Category::Keyword), + NodeKind::Break => Some(Category::Keyword), + NodeKind::Continue => Some(Category::Keyword), + NodeKind::Return => Some(Category::Keyword), + NodeKind::Import => Some(Category::Keyword), + NodeKind::Include => Some(Category::Keyword), + NodeKind::From => Some(Category::Keyword), + NodeKind::As => Some(Category::Keyword), + NodeKind::Ident(_) => match parent.kind() { NodeKind::Markup { .. } => Some(Category::Interpolated), NodeKind::FuncCall => Some(Category::Function), @@ -302,15 +338,6 @@ impl Category { NodeKind::Float(_) => Some(Category::Number), NodeKind::Numeric(_, _) => Some(Category::Number), NodeKind::Str(_) => Some(Category::String), - NodeKind::Error(_, _) => Some(Category::Invalid), - NodeKind::Unknown(_) => Some(Category::Invalid), - NodeKind::Underscore => None, - NodeKind::Markup { .. } => None, - NodeKind::Space { .. } => None, - NodeKind::Text(_) => None, - NodeKind::Quote { .. } => None, - NodeKind::List => None, - NodeKind::Enum => None, NodeKind::CodeBlock => None, NodeKind::ContentBlock => None, NodeKind::GroupExpr => None, @@ -341,6 +368,9 @@ impl Category { NodeKind::BreakExpr => None, NodeKind::ContinueExpr => None, NodeKind::ReturnExpr => None, + + NodeKind::Error(_, _) => Some(Category::Invalid), + NodeKind::Unknown(_) => Some(Category::Invalid), } } @@ -354,10 +384,12 @@ impl Category { Self::Escape => "constant.character.escape.content.typst", Self::Strong => "markup.bold.typst", Self::Emph => "markup.italic.typst", + Self::Link => "markup.underline.link.typst", Self::Raw => "markup.raw.typst", Self::Math => "string.other.math.typst", Self::Heading => "markup.heading.typst", Self::List => "markup.list.typst", + Self::Term => "markup.bold.typst", Self::Label => "entity.name.label.typst", Self::Ref => "markup.other.reference.typst", Self::Keyword => "keyword.typst", diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 89937f2cf..6c6f690ce 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -571,6 +571,14 @@ impl PartialEq for NodeData { /// the parser. #[derive(Debug, Clone, PartialEq)] pub enum NodeKind { + /// A line comment, two slashes followed by inner contents, terminated with + /// a newline: `//\n`. + LineComment, + /// A block comment, a slash and a star followed by inner contents, + /// terminated with a star and a slash: `/**/`. + /// + /// The comment can contain nested block comments. + BlockComment, /// A left curly brace, starting a code block: `{`. LeftBrace, /// A right curly brace, terminating a code block: `}`. @@ -585,23 +593,83 @@ pub enum NodeKind { /// A right round parenthesis, terminating a grouped expression, collection, /// argument or parameter list: `)`. RightParen, + + /// Markup of which all lines must have a minimal indentation. + /// + /// Notably, the number does not determine in which column the markup + /// started, but to the right of which column all markup elements must be, + /// so it is zero except for headings and lists. + Markup { min_indent: usize }, + /// One or more whitespace characters. Single spaces are collapsed into text + /// nodes if they would otherwise be surrounded by text nodes. + /// + /// Also stores how many newlines are contained. + Space { newlines: usize }, + /// A forced line break. + Linebreak, + /// Consecutive text without markup. While basic text with just single + /// spaces is collapsed into a single node, certain symbols that could + /// possibly be markup force text into multiple nodes. + Text(EcoString), + /// A slash and the letter "u" followed by a hexadecimal unicode entity + /// enclosed in curly braces: `\u{1F5FA}`. + Escape(char), + /// A non-breaking space: `~`. + NonBreakingSpace, + /// A soft hyphen: `-?`. + Shy, + /// An en-dash: `--`. + EnDash, + /// An em-dash: `---`. + EmDash, + /// An ellipsis: `...`. + Ellipsis, + /// A smart quote: `'` or `"`. + Quote { double: bool }, /// The strong text toggle, multiplication operator, and wildcard import /// symbol: `*`. Star, /// Toggles emphasized text: `_`. Underscore, + /// Strong content: `*Strong*`. + Strong, + /// Emphasized content: `_Emphasized_`. + Emph, + /// A hyperlink. + Link(EcoString), + /// A raw block with optional syntax highlighting: `` `...` ``. + Raw(Arc), + /// A math formula: `$x$`, `$[x^2]$`. + Math(Arc), + /// A section heading: `= Introduction`. + Heading, + /// An item in an unordered list: `- ...`. + List, + /// An item in an enumeration (ordered list): `+ ...` or `1. ...`. + Enum, + /// An explicit enumeration numbering: `23.`. + EnumNumbering(usize), + /// An item in a description list: `/ Term: Details. + Desc, + /// A label: `