diff --git a/Cargo.lock b/Cargo.lock index 706cd4818..74035c539 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,6 +14,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -44,6 +53,30 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -211,6 +244,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +[[package]] +name = "fancy-regex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6b8560a05112eb52f04b00e5d3790c0dd75d9d980eb8a122fb23b92a623ccf" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "filedescriptor" version = "0.8.2" @@ -240,6 +283,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fxhash" version = "0.2.1" @@ -260,6 +309,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "iai" version = "0.1.1" @@ -284,6 +339,16 @@ dependencies = [ "png 0.16.8", ] +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "itertools" version = "0.10.3" @@ -299,6 +364,12 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + [[package]] name = "jpeg-decoder" version = "0.1.22" @@ -314,12 +385,33 @@ dependencies = [ "arrayvec 0.7.2", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9" +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] + [[package]] name = "log" version = "0.4.14" @@ -335,6 +427,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + [[package]] name = "memmap2" version = "0.5.2" @@ -404,6 +502,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_threads" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" version = "1.9.0" @@ -417,7 +524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36d760a6f2ac90811cba1006a298e8a7e5ce2c922bb5dc7f7000911a4a6b60f4" dependencies = [ "bitflags", - "itoa", + "itoa 0.4.8", "ryu", ] @@ -435,6 +542,20 @@ dependencies = [ "ttf-parser", ] +[[package]] +name = "plist" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd39bc6cdc9355ad1dc5eeedefee696bb35c34caf21768741e81826c0bbd7225" +dependencies = [ + "base64", + "indexmap", + "line-wrap", + "serde", + "time", + "xml-rs", +] + [[package]] name = "png" version = "0.16.8" @@ -549,6 +670,23 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "resvg" version = "0.20.0" @@ -614,6 +752,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + [[package]] name = "same-file" version = "1.0.6" @@ -643,6 +787,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085" +dependencies = [ + "itoa 1.0.1", + "ryu", + "serde", +] + [[package]] name = "simplecss" version = "0.2.1" @@ -696,6 +851,27 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syntect" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031" +dependencies = [ + "bincode", + "bitflags", + "fancy-regex", + "flate2", + "fnv", + "lazy_static", + "lazycell", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "walkdir", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -725,6 +901,17 @@ dependencies = [ "syn", ] +[[package]] +name = "time" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d" +dependencies = [ + "itoa 1.0.1", + "libc", + "num_threads", +] + [[package]] name = "tiny-skia" version = "0.6.2" @@ -771,6 +958,7 @@ dependencies = [ "same-file", "serde", "svg2pdf", + "syntect", "tiny-skia", "ttf-parser", "typst-macros", @@ -913,6 +1101,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a" +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + [[package]] name = "xmlparser" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index bfeeb060d..2da2b02fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,9 @@ xi-unicode = "0.3" image = { version = "0.23", default-features = false, features = ["png", "jpeg"] } usvg = { version = "0.20", default-features = false } +# External implementation of user-facing features +syntect = { version = "4.6", default-features = false, features = ["dump-load", "parsing", "regex-fancy", "assets"] } + # PDF export miniz_oxide = "0.4" pdf-writer = "0.4" diff --git a/src/eval/mod.rs b/src/eval/mod.rs index c16c22083..2759e0d5a 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -30,21 +30,36 @@ use std::collections::HashMap; use std::io; use std::mem; use std::path::PathBuf; +use std::sync::Mutex; + +use once_cell::sync::Lazy; +use syntect::easy::HighlightLines; +use syntect::highlighting::{FontStyle, Highlighter, Style as SynStyle, Theme, ThemeSet}; +use syntect::parsing::SyntaxSet; use unicode_segmentation::UnicodeSegmentation; use crate::diag::{At, Error, StrResult, Trace, Tracepoint, TypResult}; -use crate::geom::{Angle, Fractional, Length, Relative}; +use crate::geom::{Angle, Fractional, Length, Paint, Relative, RgbaColor}; use crate::image::ImageStore; use crate::layout::RootNode; -use crate::library::{self, TextNode}; +use crate::library::{self, Decoration, TextNode}; use crate::loading::Loader; +use crate::parse; use crate::source::{SourceId, SourceStore}; +use crate::syntax; use crate::syntax::ast::*; -use crate::syntax::{Span, Spanned}; +use crate::syntax::{RedNode, Span, Spanned}; use crate::util::{EcoString, RefMutExt}; use crate::Context; +static THEME: Lazy> = Lazy::new(|| { + Mutex::new(ThemeSet::load_defaults().themes.remove("InspiredGitHub").unwrap()) +}); + +static SYNTAXES: Lazy> = + Lazy::new(|| Mutex::new(SyntaxSet::load_defaults_newlines())); + /// An evaluated module, ready for importing or conversion to a root layout /// tree. #[derive(Debug, Default, Clone)] @@ -209,15 +224,99 @@ impl Eval for RawNode { type Output = Node; fn eval(&self, _: &mut EvalContext) -> TypResult { - let text = Node::Text(self.text.clone()).monospaced(); + let code = self.highlighted(); Ok(if self.block { - Node::Block(text.into_block()) + Node::Block(code.into_block()) } else { - text + code }) } } +impl RawNode { + /// Styled node for a code block, with optional syntax highlighting. + pub fn highlighted(&self) -> Node { + let mut sequence: Vec> = vec![]; + let syntaxes = SYNTAXES.lock().unwrap(); + + let syntax = if let Some(syntax) = self + .lang + .as_ref() + .and_then(|token| syntaxes.find_syntax_by_token(&token)) + { + Some(syntax) + } else if matches!( + self.lang.as_ref().map(|s| s.to_ascii_lowercase()).as_deref(), + Some("typ" | "typst") + ) { + None + } else { + return Node::Text(self.text.clone()).monospaced(); + }; + + let theme = THEME.lock().unwrap(); + let foreground = theme + .settings + .foreground + .map(RgbaColor::from) + .unwrap_or(RgbaColor::BLACK) + .into(); + + match syntax { + Some(syntax) => { + let mut highlighter = HighlightLines::new(syntax, &theme); + for (i, line) in self.text.lines().enumerate() { + if i != 0 { + sequence.push(Styled::bare(Node::Linebreak)); + } + + for (style, line) in highlighter.highlight(line, &syntaxes) { + sequence.push(Self::styled_line(style, line, foreground)); + } + } + } + None => { + let red_tree = + RedNode::from_root(parse::parse(&self.text), SourceId::from_raw(0)); + let highlighter = Highlighter::new(&theme); + + syntax::highlight_syntect( + red_tree.as_ref(), + &self.text, + &highlighter, + &mut |style, line| { + sequence.push(Self::styled_line(style, line, foreground)); + }, + ) + } + } + + Node::Sequence(sequence).monospaced() + } + + fn styled_line(style: SynStyle, line: &str, foreground: Paint) -> Styled { + let paint = style.foreground.into(); + let text_node = Node::Text(line.into()); + let mut style_map = StyleMap::new(); + + if paint != foreground { + style_map.set(TextNode::FILL, paint); + } + + if style.font_style.contains(FontStyle::BOLD) { + style_map.set(TextNode::STRONG, true); + } + if style.font_style.contains(FontStyle::ITALIC) { + style_map.set(TextNode::EMPH, true); + } + if style.font_style.contains(FontStyle::UNDERLINE) { + style_map.set(TextNode::LINES, vec![Decoration::underline()]); + } + + Styled::new(text_node, style_map) + } +} + impl Eval for MathNode { type Output = Node; diff --git a/src/geom/paint.rs b/src/geom/paint.rs index d906561ce..f86386562 100644 --- a/src/geom/paint.rs +++ b/src/geom/paint.rs @@ -1,6 +1,8 @@ use std::fmt::Display; use std::str::FromStr; +use syntect::highlighting::Color as SynColor; + use super::*; /// How a fill or stroke should be painted. @@ -34,9 +36,12 @@ impl Debug for Color { } } -impl From for Color { - fn from(rgba: RgbaColor) -> Self { - Self::Rgba(rgba) +impl From for Color +where + T: Into, +{ + fn from(rgba: T) -> Self { + Self::Rgba(rgba.into()) } } @@ -114,6 +119,12 @@ impl FromStr for RgbaColor { } } +impl From for RgbaColor { + fn from(color: SynColor) -> Self { + Self::new(color.r, color.b, color.g, color.a) + } +} + impl Debug for RgbaColor { fn fmt(&self, f: &mut Formatter) -> fmt::Result { if f.alternate() { diff --git a/src/library/deco.rs b/src/library/deco.rs index 3e91d1de1..5f27c8beb 100644 --- a/src/library/deco.rs +++ b/src/library/deco.rs @@ -38,6 +38,41 @@ pub struct Decoration { pub extent: Linear, } +impl Decoration { + /// Create a new underline with default settings. + pub const fn underline() -> Self { + Self { + line: DecoLine::Underline, + stroke: None, + thickness: None, + offset: None, + extent: Linear::zero(), + } + } + + /// Create a new strikethrough with default settings. + pub const fn strikethrough() -> Self { + Self { + line: DecoLine::Underline, + stroke: None, + thickness: None, + offset: None, + extent: Linear::zero(), + } + } + + /// Create a new overline with default settings. + pub const fn overline() -> Self { + Self { + line: DecoLine::Overline, + stroke: None, + thickness: None, + offset: None, + extent: Linear::zero(), + } + } +} + /// The kind of decorative line. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum DecoLine { @@ -49,7 +84,7 @@ pub enum DecoLine { Overline, } -/// Differents kinds of decorative lines for text. +/// Different kinds of decorative lines for text. pub trait LineKind { const LINE: DecoLine; } diff --git a/src/syntax/highlight.rs b/src/syntax/highlight.rs index 9f7365a81..001b28b34 100644 --- a/src/syntax/highlight.rs +++ b/src/syntax/highlight.rs @@ -1,5 +1,8 @@ use std::ops::Range; +use syntect::highlighting::{Highlighter, Style}; +use syntect::parsing::Scope; + use super::{NodeKind, RedRef}; /// Provide highlighting categories for the children of a node that fall into a @@ -19,6 +22,45 @@ where } } +/// Provide syntect highlighting styles for the children of a node. +pub fn highlight_syntect( + node: RedRef, + text: &str, + highlighter: &Highlighter, + f: &mut F, +) where + F: FnMut(Style, &str), +{ + highlight_syntect_impl(node, text, vec![], highlighter, f) +} + +/// Recursive implementation for returning syntect styles. +fn highlight_syntect_impl( + node: RedRef, + text: &str, + scopes: Vec, + highlighter: &Highlighter, + f: &mut F, +) where + F: FnMut(Style, &str), +{ + if node.children().size_hint().0 == 0 { + f( + highlighter.style_for_stack(&scopes), + &text[node.span().to_range()], + ); + return; + } + + for child in node.children() { + let mut scopes = scopes.clone(); + if let Some(category) = Category::determine(child, node) { + scopes.push(Scope::new(category.tm_scope()).unwrap()) + } + highlight_syntect_impl(child, text, scopes, highlighter, f); + } +} + /// The syntax highlighting category of a node. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] pub enum Category { @@ -186,6 +228,33 @@ impl Category { NodeKind::IncludeExpr => None, } } + + /// Return the TextMate grammar scope for the given highlighting category. + pub const fn tm_scope(&self) -> &'static str { + match self { + Self::Bracket => "punctuation.definition.typst", + Self::Punctuation => "punctuation.typst", + Self::Comment => "comment.typst", + Self::Strong => "markup.bold.typst", + Self::Emph => "markup.italic.typst", + Self::Raw => "markup.raw.typst", + Self::Math => "string.other.math.typst", + Self::Heading => "markup.heading.typst", + Self::List => "markup.list.typst", + Self::Shortcut => "punctuation.shortcut.typst", + Self::Escape => "constant.character.escape.content.typst", + Self::Keyword => "keyword.typst", + Self::Operator => "keyword.operator.typst", + Self::None => "constant.language.none.typst", + Self::Auto => "constant.language.auto.typst", + Self::Bool => "constant.language.boolean.typst", + Self::Number => "constant.numeric.typst", + Self::String => "string.quoted.double.typst", + Self::Function => "entity.name.function.typst", + Self::Variable => "variable.parameter.typst", + Self::Invalid => "invalid.typst", + } + } } #[cfg(test)] diff --git a/tests/ref/markup/raw.png b/tests/ref/markup/raw.png index 4effb3031..ec39d3df1 100644 Binary files a/tests/ref/markup/raw.png and b/tests/ref/markup/raw.png differ