Add Code Block syntax highlighting

This commit is contained in:
Martin Haug 2021-12-14 14:24:02 +01:00
parent 3739ab7720
commit 4f66907d08
7 changed files with 422 additions and 11 deletions

196
Cargo.lock generated
View File

@ -14,6 +14,15 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aho-corasick"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
[[package]]
name = "arrayref"
version = "0.3.6"
@ -44,6 +53,30 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bit-set"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -211,6 +244,16 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
[[package]]
name = "fancy-regex"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d6b8560a05112eb52f04b00e5d3790c0dd75d9d980eb8a122fb23b92a623ccf"
dependencies = [
"bit-set",
"regex",
]
[[package]]
name = "filedescriptor"
version = "0.8.2"
@ -240,6 +283,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fxhash"
version = "0.2.1"
@ -260,6 +309,12 @@ dependencies = [
"wasi",
]
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
[[package]]
name = "iai"
version = "0.1.1"
@ -284,6 +339,16 @@ dependencies = [
"png 0.16.8",
]
[[package]]
name = "indexmap"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "itertools"
version = "0.10.3"
@ -299,6 +364,12 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "jpeg-decoder"
version = "0.1.22"
@ -314,12 +385,33 @@ dependencies = [
"arrayvec 0.7.2",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.113"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9"
[[package]]
name = "line-wrap"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
dependencies = [
"safemem",
]
[[package]]
name = "log"
version = "0.4.14"
@ -335,6 +427,12 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "memchr"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memmap2"
version = "0.5.2"
@ -404,6 +502,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_threads"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15"
dependencies = [
"libc",
]
[[package]]
name = "once_cell"
version = "1.9.0"
@ -417,7 +524,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36d760a6f2ac90811cba1006a298e8a7e5ce2c922bb5dc7f7000911a4a6b60f4"
dependencies = [
"bitflags",
"itoa",
"itoa 0.4.8",
"ryu",
]
@ -435,6 +542,20 @@ dependencies = [
"ttf-parser",
]
[[package]]
name = "plist"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd39bc6cdc9355ad1dc5eeedefee696bb35c34caf21768741e81826c0bbd7225"
dependencies = [
"base64",
"indexmap",
"line-wrap",
"serde",
"time",
"xml-rs",
]
[[package]]
name = "png"
version = "0.16.8"
@ -549,6 +670,23 @@ dependencies = [
"redox_syscall",
]
[[package]]
name = "regex"
version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "resvg"
version = "0.20.0"
@ -614,6 +752,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "safemem"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
[[package]]
name = "same-file"
version = "1.0.6"
@ -643,6 +787,17 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085"
dependencies = [
"itoa 1.0.1",
"ryu",
"serde",
]
[[package]]
name = "simplecss"
version = "0.2.1"
@ -696,6 +851,27 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "syntect"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031"
dependencies = [
"bincode",
"bitflags",
"fancy-regex",
"flate2",
"fnv",
"lazy_static",
"lazycell",
"plist",
"regex-syntax",
"serde",
"serde_derive",
"serde_json",
"walkdir",
]
[[package]]
name = "termcolor"
version = "1.1.2"
@ -725,6 +901,17 @@ dependencies = [
"syn",
]
[[package]]
name = "time"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d"
dependencies = [
"itoa 1.0.1",
"libc",
"num_threads",
]
[[package]]
name = "tiny-skia"
version = "0.6.2"
@ -771,6 +958,7 @@ dependencies = [
"same-file",
"serde",
"svg2pdf",
"syntect",
"tiny-skia",
"ttf-parser",
"typst-macros",
@ -913,6 +1101,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a67300977d3dc3f8034dae89778f502b6ba20b269527b3223ba59c0cf393bb8a"
[[package]]
name = "xml-rs"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
[[package]]
name = "xmlparser"
version = "0.13.3"

View File

@ -37,6 +37,9 @@ xi-unicode = "0.3"
image = { version = "0.23", default-features = false, features = ["png", "jpeg"] }
usvg = { version = "0.20", default-features = false }
# External implementation of user-facing features
syntect = { version = "4.6", default-features = false, features = ["dump-load", "parsing", "regex-fancy", "assets"] }
# PDF export
miniz_oxide = "0.4"
pdf-writer = "0.4"

View File

@ -30,21 +30,36 @@ use std::collections::HashMap;
use std::io;
use std::mem;
use std::path::PathBuf;
use std::sync::Mutex;
use once_cell::sync::Lazy;
use syntect::easy::HighlightLines;
use syntect::highlighting::{FontStyle, Highlighter, Style as SynStyle, Theme, ThemeSet};
use syntect::parsing::SyntaxSet;
use unicode_segmentation::UnicodeSegmentation;
use crate::diag::{At, Error, StrResult, Trace, Tracepoint, TypResult};
use crate::geom::{Angle, Fractional, Length, Relative};
use crate::geom::{Angle, Fractional, Length, Paint, Relative, RgbaColor};
use crate::image::ImageStore;
use crate::layout::RootNode;
use crate::library::{self, TextNode};
use crate::library::{self, Decoration, TextNode};
use crate::loading::Loader;
use crate::parse;
use crate::source::{SourceId, SourceStore};
use crate::syntax;
use crate::syntax::ast::*;
use crate::syntax::{Span, Spanned};
use crate::syntax::{RedNode, Span, Spanned};
use crate::util::{EcoString, RefMutExt};
use crate::Context;
static THEME: Lazy<Mutex<Theme>> = Lazy::new(|| {
Mutex::new(ThemeSet::load_defaults().themes.remove("InspiredGitHub").unwrap())
});
static SYNTAXES: Lazy<Mutex<SyntaxSet>> =
Lazy::new(|| Mutex::new(SyntaxSet::load_defaults_newlines()));
/// An evaluated module, ready for importing or conversion to a root layout
/// tree.
#[derive(Debug, Default, Clone)]
@ -209,15 +224,99 @@ impl Eval for RawNode {
type Output = Node;
fn eval(&self, _: &mut EvalContext) -> TypResult<Self::Output> {
let text = Node::Text(self.text.clone()).monospaced();
let code = self.highlighted();
Ok(if self.block {
Node::Block(text.into_block())
Node::Block(code.into_block())
} else {
text
code
})
}
}
impl RawNode {
/// Styled node for a code block, with optional syntax highlighting.
pub fn highlighted(&self) -> Node {
let mut sequence: Vec<Styled<Node>> = vec![];
let syntaxes = SYNTAXES.lock().unwrap();
let syntax = if let Some(syntax) = self
.lang
.as_ref()
.and_then(|token| syntaxes.find_syntax_by_token(&token))
{
Some(syntax)
} else if matches!(
self.lang.as_ref().map(|s| s.to_ascii_lowercase()).as_deref(),
Some("typ" | "typst")
) {
None
} else {
return Node::Text(self.text.clone()).monospaced();
};
let theme = THEME.lock().unwrap();
let foreground = theme
.settings
.foreground
.map(RgbaColor::from)
.unwrap_or(RgbaColor::BLACK)
.into();
match syntax {
Some(syntax) => {
let mut highlighter = HighlightLines::new(syntax, &theme);
for (i, line) in self.text.lines().enumerate() {
if i != 0 {
sequence.push(Styled::bare(Node::Linebreak));
}
for (style, line) in highlighter.highlight(line, &syntaxes) {
sequence.push(Self::styled_line(style, line, foreground));
}
}
}
None => {
let red_tree =
RedNode::from_root(parse::parse(&self.text), SourceId::from_raw(0));
let highlighter = Highlighter::new(&theme);
syntax::highlight_syntect(
red_tree.as_ref(),
&self.text,
&highlighter,
&mut |style, line| {
sequence.push(Self::styled_line(style, line, foreground));
},
)
}
}
Node::Sequence(sequence).monospaced()
}
fn styled_line(style: SynStyle, line: &str, foreground: Paint) -> Styled<Node> {
let paint = style.foreground.into();
let text_node = Node::Text(line.into());
let mut style_map = StyleMap::new();
if paint != foreground {
style_map.set(TextNode::FILL, paint);
}
if style.font_style.contains(FontStyle::BOLD) {
style_map.set(TextNode::STRONG, true);
}
if style.font_style.contains(FontStyle::ITALIC) {
style_map.set(TextNode::EMPH, true);
}
if style.font_style.contains(FontStyle::UNDERLINE) {
style_map.set(TextNode::LINES, vec![Decoration::underline()]);
}
Styled::new(text_node, style_map)
}
}
impl Eval for MathNode {
type Output = Node;

View File

@ -1,6 +1,8 @@
use std::fmt::Display;
use std::str::FromStr;
use syntect::highlighting::Color as SynColor;
use super::*;
/// How a fill or stroke should be painted.
@ -34,9 +36,12 @@ impl Debug for Color {
}
}
impl From<RgbaColor> for Color {
fn from(rgba: RgbaColor) -> Self {
Self::Rgba(rgba)
impl<T> From<T> for Color
where
T: Into<RgbaColor>,
{
fn from(rgba: T) -> Self {
Self::Rgba(rgba.into())
}
}
@ -114,6 +119,12 @@ impl FromStr for RgbaColor {
}
}
impl From<SynColor> for RgbaColor {
fn from(color: SynColor) -> Self {
Self::new(color.r, color.b, color.g, color.a)
}
}
impl Debug for RgbaColor {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
if f.alternate() {

View File

@ -38,6 +38,41 @@ pub struct Decoration {
pub extent: Linear,
}
impl Decoration {
/// Create a new underline with default settings.
pub const fn underline() -> Self {
Self {
line: DecoLine::Underline,
stroke: None,
thickness: None,
offset: None,
extent: Linear::zero(),
}
}
/// Create a new strikethrough with default settings.
pub const fn strikethrough() -> Self {
Self {
line: DecoLine::Underline,
stroke: None,
thickness: None,
offset: None,
extent: Linear::zero(),
}
}
/// Create a new overline with default settings.
pub const fn overline() -> Self {
Self {
line: DecoLine::Overline,
stroke: None,
thickness: None,
offset: None,
extent: Linear::zero(),
}
}
}
/// The kind of decorative line.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum DecoLine {
@ -49,7 +84,7 @@ pub enum DecoLine {
Overline,
}
/// Differents kinds of decorative lines for text.
/// Different kinds of decorative lines for text.
pub trait LineKind {
const LINE: DecoLine;
}

View File

@ -1,5 +1,8 @@
use std::ops::Range;
use syntect::highlighting::{Highlighter, Style};
use syntect::parsing::Scope;
use super::{NodeKind, RedRef};
/// Provide highlighting categories for the children of a node that fall into a
@ -19,6 +22,45 @@ where
}
}
/// Provide syntect highlighting styles for the children of a node.
pub fn highlight_syntect<F>(
node: RedRef,
text: &str,
highlighter: &Highlighter,
f: &mut F,
) where
F: FnMut(Style, &str),
{
highlight_syntect_impl(node, text, vec![], highlighter, f)
}
/// Recursive implementation for returning syntect styles.
fn highlight_syntect_impl<F>(
node: RedRef,
text: &str,
scopes: Vec<Scope>,
highlighter: &Highlighter,
f: &mut F,
) where
F: FnMut(Style, &str),
{
if node.children().size_hint().0 == 0 {
f(
highlighter.style_for_stack(&scopes),
&text[node.span().to_range()],
);
return;
}
for child in node.children() {
let mut scopes = scopes.clone();
if let Some(category) = Category::determine(child, node) {
scopes.push(Scope::new(category.tm_scope()).unwrap())
}
highlight_syntect_impl(child, text, scopes, highlighter, f);
}
}
/// The syntax highlighting category of a node.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum Category {
@ -186,6 +228,33 @@ impl Category {
NodeKind::IncludeExpr => None,
}
}
/// Return the TextMate grammar scope for the given highlighting category.
pub const fn tm_scope(&self) -> &'static str {
match self {
Self::Bracket => "punctuation.definition.typst",
Self::Punctuation => "punctuation.typst",
Self::Comment => "comment.typst",
Self::Strong => "markup.bold.typst",
Self::Emph => "markup.italic.typst",
Self::Raw => "markup.raw.typst",
Self::Math => "string.other.math.typst",
Self::Heading => "markup.heading.typst",
Self::List => "markup.list.typst",
Self::Shortcut => "punctuation.shortcut.typst",
Self::Escape => "constant.character.escape.content.typst",
Self::Keyword => "keyword.typst",
Self::Operator => "keyword.operator.typst",
Self::None => "constant.language.none.typst",
Self::Auto => "constant.language.auto.typst",
Self::Bool => "constant.language.boolean.typst",
Self::Number => "constant.numeric.typst",
Self::String => "string.quoted.double.typst",
Self::Function => "entity.name.function.typst",
Self::Variable => "variable.parameter.typst",
Self::Invalid => "invalid.typst",
}
}
}
#[cfg(test)]

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.9 KiB

After

Width:  |  Height:  |  Size: 22 KiB