konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move highlighting into new IDE module

Browse Source
This commit is contained in:
Laurenz 2022-12-14 10:00:32 +01:00
parent 2470df05af
commit 4c73456fc1
12 changed files with 392 additions and 508 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1150,7 +1150,6 @@ dependencies = [
"siphasher",
"subsetter",
"svg2pdf",
"syntect",
"thin-vec",
"tiny-skia",
"ttf-parser 0.17.1",

1
Cargo.toml
View File

@ -31,7 +31,6 @@ serde = { version = "1", features = ["derive"] }
siphasher = "0.3"
subsetter = "0.1"
svg2pdf = "0.4"
syntect = { version = "5", default-features = false, features = ["default-syntaxes", "regex-fancy"] }
thin-vec = "0.2"
tiny-skia = "0.6.2"
ttf-parser = "0.17"

108
library/src/text/raw.rs
View File

@ -1,10 +1,6 @@
use once_cell::sync::Lazy;
use syntect::easy::HighlightLines;
use syntect::highlighting::{
Color, FontStyle, Style, StyleModifier, Theme, ThemeItem, ThemeSettings,
};
use syntect::parsing::SyntaxSet;
use typst::syntax;
use syntect::highlighting as synt;
use typst::syntax::{self, LinkedNode};
use super::{FontFamily, Hyphenate, LinebreakNode, TextNode};
use crate::layout::BlockNode;
@ -61,8 +57,8 @@ impl Show for RawNode {
let foreground = THEME
.settings
.foreground
.map(Color::from)
.unwrap_or(Color::BLACK)
.map(to_typst)
.map_or(Color::BLACK, Color::from)
.into();
let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) {
@ -72,16 +68,22 @@ impl Show for RawNode {
};
let mut seq = vec![];
syntax::highlight::highlight_themed(&root, &THEME, |range, style| {
seq.push(styled(&self.text[range], foreground, style));
});
let highlighter = synt::Highlighter::new(&THEME);
highlight_themed(
&LinkedNode::new(&root),
vec![],
&highlighter,
&mut |node, style| {
seq.push(styled(&self.text[node.range()], foreground, style));
},
);
Content::sequence(seq)
} else if let Some(syntax) =
lang.and_then(|token| SYNTAXES.find_syntax_by_token(&token))
{
let mut seq = vec![];
let mut highlighter = HighlightLines::new(syntax, &THEME);
let mut highlighter = syntect::easy::HighlightLines::new(syntax, &THEME);
for (i, line) in self.text.lines().enumerate() {
if i != 0 {
seq.push(LinebreakNode { justify: false }.pack());
@ -113,54 +115,90 @@ impl Show for RawNode {
}
}
/// Highlight a syntax node in a theme by calling `f` with ranges and their
/// styles.
fn highlight_themed<F>(
node: &LinkedNode,
scopes: Vec<syntect::parsing::Scope>,
highlighter: &synt::Highlighter,
f: &mut F,
) where
F: FnMut(&LinkedNode, synt::Style),
{
if node.children().len() == 0 {
let style = highlighter.style_for_stack(&scopes);
f(node, style);
return;
}
for child in node.children() {
let mut scopes = scopes.clone();
if let Some(tag) = typst::ide::highlight(&child) {
scopes.push(syntect::parsing::Scope::new(tag.tm_scope()).unwrap())
}
highlight_themed(&child, scopes, highlighter, f);
}
}
/// Style a piece of text with a syntect style.
fn styled(piece: &str, foreground: Paint, style: Style) -> Content {
fn styled(piece: &str, foreground: Paint, style: synt::Style) -> Content {
let mut body = TextNode::packed(piece);
let paint = style.foreground.into();
let paint = to_typst(style.foreground).into();
if paint != foreground {
body = body.styled(TextNode::FILL, paint);
}
if style.font_style.contains(FontStyle::BOLD) {
if style.font_style.contains(synt::FontStyle::BOLD) {
body = body.strong();
}
if style.font_style.contains(FontStyle::ITALIC) {
if style.font_style.contains(synt::FontStyle::ITALIC) {
body = body.emph();
}
if style.font_style.contains(FontStyle::UNDERLINE) {
if style.font_style.contains(synt::FontStyle::UNDERLINE) {
body = body.underlined();
}
body
}
fn to_typst(synt::Color { r, g, b, a }: synt::Color) -> RgbaColor {
RgbaColor { r, g, b, a }
}
fn to_syn(RgbaColor { r, g, b, a }: RgbaColor) -> synt::Color {
synt::Color { r, g, b, a }
}
/// The syntect syntax definitions.
static SYNTAXES: Lazy<SyntaxSet> = Lazy::new(|| SyntaxSet::load_defaults_newlines());
static SYNTAXES: Lazy<syntect::parsing::SyntaxSet> =
Lazy::new(|| syntect::parsing::SyntaxSet::load_defaults_newlines());
/// The default theme used for syntax highlighting.
#[rustfmt::skip]
pub static THEME: Lazy<Theme> = Lazy::new(|| Theme {
pub static THEME: Lazy<synt::Theme> = Lazy::new(|| synt::Theme {
name: Some("Typst Light".into()),
author: Some("The Typst Project Developers".into()),
settings: ThemeSettings::default(),
settings: synt::ThemeSettings::default(),
scopes: vec![
item("comment", Some("#8a8a8a"), None),
item("constant.character.escape", Some("#1d6c76"), None),
item("constant.character.shortcut", Some("#1d6c76"), None),
item("markup.bold", None, Some(FontStyle::BOLD)),
item("markup.italic", None, Some(FontStyle::ITALIC)),
item("markup.underline", None, Some(FontStyle::UNDERLINE)),
item("markup.bold", None, Some(synt::FontStyle::BOLD)),
item("markup.italic", None, Some(synt::FontStyle::ITALIC)),
item("markup.underline", None, Some(synt::FontStyle::UNDERLINE)),
item("markup.raw", Some("#818181"), None),
item("string.other.math.typst", None, None),
item("punctuation.definition.math", Some("#298e0d"), None),
item("keyword.operator.math", Some("#1d6c76"), None),
item("markup.heading, entity.name.section", None, Some(FontStyle::BOLD)),
item("markup.heading.typst", None, Some(FontStyle::BOLD | FontStyle::UNDERLINE)),
item("markup.heading, entity.name.section", None, Some(synt::FontStyle::BOLD)),
item(
"markup.heading.typst",
None,
Some(synt::FontStyle::BOLD | synt::FontStyle::UNDERLINE),
),
item("punctuation.definition.list", Some("#8b41b1"), None),
item("markup.list.term", None, Some(FontStyle::BOLD)),
item("markup.list.term", None, Some(synt::FontStyle::BOLD)),
item("entity.name.label, markup.other.reference", Some("#1d6c76"), None),
item("keyword, constant.language, variable.language", Some("#d73a49"), None),
item("storage.type, storage.modifier", Some("#d73a49"), None),
@ -169,17 +207,21 @@ pub static THEME: Lazy<Theme> = Lazy::new(|| Theme {
item("entity.name, variable.function, support", Some("#4b69c6"), None),
item("support.macro", Some("#16718d"), None),
item("meta.annotation", Some("#301414"), None),
item("entity.other, meta.interpolation, constant.symbol.typst", Some("#8b41b1"), None),
item("entity.other, meta.interpolation", Some("#8b41b1"), None),
item("invalid", Some("#ff0000"), None),
],
});
/// Create a syntect theme item.
fn item(scope: &str, color: Option<&str>, font_style: Option<FontStyle>) -> ThemeItem {
ThemeItem {
fn item(
scope: &str,
color: Option<&str>,
font_style: Option<synt::FontStyle>,
) -> synt::ThemeItem {
synt::ThemeItem {
scope: scope.parse().unwrap(),
style: StyleModifier {
foreground: color.map(|s| s.parse::<RgbaColor>().unwrap().into()),
style: synt::StyleModifier {
foreground: color.map(|s| to_syn(s.parse::<RgbaColor>().unwrap())),
background: None,
font_style,
},

14
src/geom/paint.rs
View File

@ -1,7 +1,5 @@
use std::str::FromStr;
use syntect::highlighting::Color as SynColor;
use super::*;
/// How a fill or stroke should be painted.
@ -261,18 +259,6 @@ impl FromStr for RgbaColor {
}
}
impl From<SynColor> for RgbaColor {
fn from(SynColor { r, g, b, a }: SynColor) -> Self {
Self { r, g, b, a }
}
}
impl From<RgbaColor> for SynColor {
fn from(RgbaColor { r, g, b, a }: RgbaColor) -> Self {
Self { r, g, b, a }
}
}
impl Debug for RgbaColor {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
if f.alternate() {

308
src/ide/highlight.rs Normal file
View File

@ -0,0 +1,308 @@
//! Syntax highlighting for Typst source code.
use crate::syntax::{LinkedNode, SyntaxKind};
/// Syntax highlighting categories.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum Category {
/// A line or block comment.
Comment,
/// Punctuation in code.
Punctuation,
/// An escape sequence, shorthand or symbol notation.
Escape,
/// Strong markup.
Strong,
/// Emphasized markup.
Emph,
/// A hyperlink.
Link,
/// Raw text.
Raw,
/// A label.
Label,
/// A reference to a label.
Ref,
/// A section heading.
Heading,
/// A marker of a list, enumeration, or description list.
ListMarker,
/// A term in a description list.
ListTerm,
/// The delimiters of a math formula.
MathDelimiter,
/// An operator with special meaning in a math formula.
MathOperator,
/// A keyword.
Keyword,
/// An operator in code.
Operator,
/// A numeric literal.
Number,
/// A string literal.
String,
/// A function or method name.
Function,
/// An interpolated variable in markup or math.
Interpolated,
/// A syntax error.
Error,
}
impl Category {
/// Return the recommended TextMate grammar scope for the given highlighting
/// category.
pub fn tm_scope(&self) -> &'static str {
match self {
Self::Comment => "comment.typst",
Self::Punctuation => "punctuation.typst",
Self::Escape => "constant.character.escape.typst",
Self::Strong => "markup.bold.typst",
Self::Emph => "markup.italic.typst",
Self::Link => "markup.underline.link.typst",
Self::Raw => "markup.raw.typst",
Self::MathDelimiter => "punctuation.definition.math.typst",
Self::MathOperator => "keyword.operator.math.typst",
Self::Heading => "markup.heading.typst",
Self::ListMarker => "punctuation.definition.list.typst",
Self::ListTerm => "markup.list.term.typst",
Self::Label => "entity.name.label.typst",
Self::Ref => "markup.other.reference.typst",
Self::Keyword => "keyword.typst",
Self::Operator => "keyword.operator.typst",
Self::Number => "constant.numeric.typst",
Self::String => "string.quoted.double.typst",
Self::Function => "entity.name.function.typst",
Self::Interpolated => "meta.interpolation.typst",
Self::Error => "invalid.typst",
}
}
}
/// Highlight a linked syntax node.
///
/// Produces a highlighting category or `None` if the node should not be
/// highlighted.
pub fn highlight(node: &LinkedNode) -> Option<Category> {
match node.kind() {
SyntaxKind::LineComment => Some(Category::Comment),
SyntaxKind::BlockComment => Some(Category::Comment),
SyntaxKind::Space { .. } => None,
SyntaxKind::LeftBrace => Some(Category::Punctuation),
SyntaxKind::RightBrace => Some(Category::Punctuation),
SyntaxKind::LeftBracket => Some(Category::Punctuation),
SyntaxKind::RightBracket => Some(Category::Punctuation),
SyntaxKind::LeftParen => Some(Category::Punctuation),
SyntaxKind::RightParen => Some(Category::Punctuation),
SyntaxKind::Comma => Some(Category::Punctuation),
SyntaxKind::Semicolon => Some(Category::Punctuation),
SyntaxKind::Colon => Some(Category::Punctuation),
SyntaxKind::Star => match node.parent_kind() {
Some(SyntaxKind::Strong) => None,
_ => Some(Category::Operator),
},
SyntaxKind::Underscore => match node.parent_kind() {
Some(SyntaxKind::Script) => Some(Category::MathOperator),
_ => None,
},
SyntaxKind::Dollar => Some(Category::MathDelimiter),
SyntaxKind::Plus => Some(match node.parent_kind() {
Some(SyntaxKind::EnumItem) => Category::ListMarker,
_ => Category::Operator,
}),
SyntaxKind::Minus => Some(match node.parent_kind() {
Some(SyntaxKind::ListItem) => Category::ListMarker,
_ => Category::Operator,
}),
SyntaxKind::Slash => Some(match node.parent_kind() {
Some(SyntaxKind::DescItem) => Category::ListMarker,
Some(SyntaxKind::Frac) => Category::MathOperator,
_ => Category::Operator,
}),
SyntaxKind::Hat => Some(Category::MathOperator),
SyntaxKind::Amp => Some(Category::MathOperator),
SyntaxKind::Dot => Some(Category::Punctuation),
SyntaxKind::Eq => match node.parent_kind() {
Some(SyntaxKind::Heading) => None,
_ => Some(Category::Operator),
},
SyntaxKind::EqEq => Some(Category::Operator),
SyntaxKind::ExclEq => Some(Category::Operator),
SyntaxKind::Lt => Some(Category::Operator),
SyntaxKind::LtEq => Some(Category::Operator),
SyntaxKind::Gt => Some(Category::Operator),
SyntaxKind::GtEq => Some(Category::Operator),
SyntaxKind::PlusEq => Some(Category::Operator),
SyntaxKind::HyphEq => Some(Category::Operator),
SyntaxKind::StarEq => Some(Category::Operator),
SyntaxKind::SlashEq => Some(Category::Operator),
SyntaxKind::Dots => Some(Category::Operator),
SyntaxKind::Arrow => Some(Category::Operator),
SyntaxKind::Not => Some(Category::Keyword),
SyntaxKind::And => Some(Category::Keyword),
SyntaxKind::Or => Some(Category::Keyword),
SyntaxKind::None => Some(Category::Keyword),
SyntaxKind::Auto => Some(Category::Keyword),
SyntaxKind::Let => Some(Category::Keyword),
SyntaxKind::Set => Some(Category::Keyword),
SyntaxKind::Show => Some(Category::Keyword),
SyntaxKind::If => Some(Category::Keyword),
SyntaxKind::Else => Some(Category::Keyword),
SyntaxKind::For => Some(Category::Keyword),
SyntaxKind::In => Some(Category::Keyword),
SyntaxKind::While => Some(Category::Keyword),
SyntaxKind::Break => Some(Category::Keyword),
SyntaxKind::Continue => Some(Category::Keyword),
SyntaxKind::Return => Some(Category::Keyword),
SyntaxKind::Import => Some(Category::Keyword),
SyntaxKind::Include => Some(Category::Keyword),
SyntaxKind::From => Some(Category::Keyword),
SyntaxKind::Markup { .. }
if node.parent_kind() == Some(&SyntaxKind::DescItem)
&& node.next_sibling_kind() == Some(&SyntaxKind::Colon) =>
{
Some(Category::ListTerm)
}
SyntaxKind::Markup { .. } => None,
SyntaxKind::Text(_) => None,
SyntaxKind::Linebreak => Some(Category::Escape),
SyntaxKind::Escape(_) => Some(Category::Escape),
SyntaxKind::Shorthand(_) => Some(Category::Escape),
SyntaxKind::Symbol(_) => Some(Category::Escape),
SyntaxKind::SmartQuote { .. } => None,
SyntaxKind::Strong => Some(Category::Strong),
SyntaxKind::Emph => Some(Category::Emph),
SyntaxKind::Raw(_) => Some(Category::Raw),
SyntaxKind::Link(_) => Some(Category::Link),
SyntaxKind::Label(_) => Some(Category::Label),
SyntaxKind::Ref(_) => Some(Category::Ref),
SyntaxKind::Heading => Some(Category::Heading),
SyntaxKind::ListItem => None,
SyntaxKind::EnumItem => None,
SyntaxKind::EnumNumbering(_) => Some(Category::ListMarker),
SyntaxKind::DescItem => None,
SyntaxKind::Math => None,
SyntaxKind::Atom(_) => None,
SyntaxKind::Script => None,
SyntaxKind::Frac => None,
SyntaxKind::AlignPoint => None,
SyntaxKind::Ident(_) => match node.parent_kind() {
Some(
SyntaxKind::Markup { .. }
| SyntaxKind::Math
| SyntaxKind::Script
| SyntaxKind::Frac,
) => Some(Category::Interpolated),
Some(SyntaxKind::FuncCall) => Some(Category::Function),
Some(SyntaxKind::MethodCall) if node.prev_sibling().is_some() => {
Some(Category::Function)
}
Some(SyntaxKind::Closure) if node.prev_sibling().is_none() => {
Some(Category::Function)
}
Some(SyntaxKind::SetRule) => Some(Category::Function),
Some(SyntaxKind::ShowRule)
if node.prev_sibling_kind() == Some(&SyntaxKind::Show) =>
{
Some(Category::Function)
}
_ => None,
},
SyntaxKind::Bool(_) => Some(Category::Keyword),
SyntaxKind::Int(_) => Some(Category::Number),
SyntaxKind::Float(_) => Some(Category::Number),
SyntaxKind::Numeric(_, _) => Some(Category::Number),
SyntaxKind::Str(_) => Some(Category::String),
SyntaxKind::CodeBlock => None,
SyntaxKind::ContentBlock => None,
SyntaxKind::Parenthesized => None,
SyntaxKind::Array => None,
SyntaxKind::Dict => None,
SyntaxKind::Named => None,
SyntaxKind::Keyed => None,
SyntaxKind::Unary => None,
SyntaxKind::Binary => None,
SyntaxKind::FieldAccess => None,
SyntaxKind::FuncCall => None,
SyntaxKind::MethodCall => None,
SyntaxKind::Args => None,
SyntaxKind::Spread => None,
SyntaxKind::Closure => None,
SyntaxKind::Params => None,
SyntaxKind::LetBinding => None,
SyntaxKind::SetRule => None,
SyntaxKind::ShowRule => None,
SyntaxKind::Conditional => None,
SyntaxKind::WhileLoop => None,
SyntaxKind::ForLoop => None,
SyntaxKind::ForPattern => None,
SyntaxKind::ModuleImport => None,
SyntaxKind::ImportItems => None,
SyntaxKind::ModuleInclude => None,
SyntaxKind::LoopBreak => None,
SyntaxKind::LoopContinue => None,
SyntaxKind::FuncReturn => None,
SyntaxKind::Error(_, _) => Some(Category::Error),
}
}
#[cfg(test)]
mod tests {
use std::ops::Range;
use super::*;
use crate::syntax::Source;
#[test]
fn test_highlighting() {
use Category::*;
#[track_caller]
fn test(text: &str, goal: &[(Range<usize>, Category)]) {
let mut vec = vec![];
let source = Source::detached(text);
highlight_tree(&mut vec, &LinkedNode::new(source.root()));
assert_eq!(vec, goal);
}
fn highlight_tree(tags: &mut Vec<(Range<usize>, Category)>, node: &LinkedNode) {
if let Some(tag) = highlight(node) {
tags.push((node.range(), tag));
}
for child in node.children() {
highlight_tree(tags, &child);
}
}
test("= *AB*", &[(0..6, Heading), (2..6, Strong)]);
test(
"#f(x + 1)",
&[
(0..2, Function),
(2..3, Punctuation),
(5..6, Operator),
(7..8, Number),
(8..9, Punctuation),
],
);
test(
"#let f(x) = x",
&[
(0..4, Keyword),
(5..6, Function),
(6..7, Punctuation),
(8..9, Punctuation),
(10..11, Operator),
],
);
}
}

5
src/ide/mod.rs Normal file
View File

@ -0,0 +1,5 @@
//! Capabilities for IDE support.
mod highlight;
pub use highlight::*;

1
src/lib.rs
View File

@ -42,6 +42,7 @@ pub mod model;
pub mod doc;
pub mod export;
pub mod font;
pub mod ide;
pub mod image;
pub mod syntax;

445
src/syntax/highlight.rs
View File

@ -1,445 +0,0 @@
//! Syntax highlighting for Typst source code.
use std::fmt::Write;
use std::ops::Range;
use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme};
use syntect::parsing::Scope;
use super::{parse, SyntaxKind, SyntaxNode};
/// Highlight source text into a standalone HTML document.
pub fn highlight_html(text: &str, theme: &Theme) -> String {
let mut buf = String::new();
buf.push_str("<!DOCTYPE html>\n");
buf.push_str("<html>\n");
buf.push_str("<head>\n");
buf.push_str(" <meta charset=\"utf-8\">\n");
buf.push_str("</head>\n");
buf.push_str("<body>\n");
buf.push_str(&highlight_pre(text, theme));
buf.push_str("\n</body>\n");
buf.push_str("</html>\n");
buf
}
/// Highlight source text into an HTML pre element.
pub fn highlight_pre(text: &str, theme: &Theme) -> String {
let mut buf = String::new();
buf.push_str("<pre>\n");
let root = parse(text);
highlight_themed(&root, theme, |range, style| {
let styled = style != Style::default();
if styled {
buf.push_str("<span style=\"");
if style.foreground != Color::BLACK {
let Color { r, g, b, a } = style.foreground;
write!(buf, "color: #{r:02x}{g:02x}{b:02x}{a:02x};").unwrap();
}
if style.font_style.contains(FontStyle::BOLD) {
buf.push_str("font-weight:bold;");
}
if style.font_style.contains(FontStyle::ITALIC) {
buf.push_str("font-style:italic;");
}
if style.font_style.contains(FontStyle::UNDERLINE) {
buf.push_str("text-decoration:underline;")
}
buf.push_str("\">");
}
buf.push_str(&text[range]);
if styled {
buf.push_str("</span>");
}
});
buf.push_str("\n</pre>");
buf
}
/// Highlight a syntax node in a theme by calling `f` with ranges and their
/// styles.
pub fn highlight_themed<F>(root: &SyntaxNode, theme: &Theme, mut f: F)
where
F: FnMut(Range<usize>, Style),
{
fn process<F>(
mut offset: usize,
node: &SyntaxNode,
scopes: Vec<Scope>,
highlighter: &Highlighter,
f: &mut F,
) where
F: FnMut(Range<usize>, Style),
{
if node.children().len() == 0 {
let range = offset..offset + node.len();
let style = highlighter.style_for_stack(&scopes);
f(range, style);
return;
}
for (i, child) in node.children().enumerate() {
let mut scopes = scopes.clone();
if let Some(category) = Category::determine(child, node, i) {
scopes.push(Scope::new(category.tm_scope()).unwrap())
}
process(offset, child, scopes, highlighter, f);
offset += child.len();
}
}
let highlighter = Highlighter::new(theme);
process(0, root, vec![], &highlighter, &mut f);
}
/// Highlight a syntax node by calling `f` with ranges overlapping `within` and
/// their categories.
pub fn highlight_categories<F>(root: &SyntaxNode, within: Range<usize>, mut f: F)
where
F: FnMut(Range<usize>, Category),
{
fn process<F>(mut offset: usize, node: &SyntaxNode, range: Range<usize>, f: &mut F)
where
F: FnMut(Range<usize>, Category),
{
for (i, child) in node.children().enumerate() {
let span = offset..offset + child.len();
if range.start <= span.end && range.end >= span.start {
if let Some(category) = Category::determine(child, node, i) {
f(span, category);
}
process(offset, child, range.clone(), f);
}
offset += child.len();
}
}
process(0, root, within, &mut f)
}
/// The syntax highlighting category of a node.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub enum Category {
/// A line or block comment.
Comment,
/// A square bracket, parenthesis or brace.
Bracket,
/// Punctuation in code.
Punctuation,
/// An escape sequence.
Escape,
/// An easily typable shortcut to a unicode codepoint.
Shorthand,
/// Symbol notation.
Symbol,
/// A smart quote.
SmartQuote,
/// Strong markup.
Strong,
/// Emphasized markup.
Emph,
/// A hyperlink.
Link,
/// Raw text.
Raw,
/// A label.
Label,
/// A reference.
Ref,
/// A section heading.
Heading,
/// A full item of a list, enumeration or description list.
ListItem,
/// A marker of a list, enumeration, or description list.
ListMarker,
/// A term in a description list.
ListTerm,
/// The delimiters of a math formula.
MathDelimiter,
/// An operator with special meaning in a math formula.
MathOperator,
/// A keyword.
Keyword,
/// A literal defined by a keyword like `none`, `auto` or a boolean.
KeywordLiteral,
/// An operator symbol.
Operator,
/// A numeric literal.
Number,
/// A string literal.
String,
/// A function or method name.
Function,
/// An interpolated variable in markup or math.
Interpolated,
/// A syntax error.
Error,
}
impl Category {
/// Determine the highlighting category of a node given its parent and its
/// index in its siblings.
pub fn determine(
child: &SyntaxNode,
parent: &SyntaxNode,
i: usize,
) -> Option<Category> {
match child.kind() {
SyntaxKind::LineComment => Some(Category::Comment),
SyntaxKind::BlockComment => Some(Category::Comment),
SyntaxKind::Space { .. } => None,
SyntaxKind::LeftBrace => Some(Category::Bracket),
SyntaxKind::RightBrace => Some(Category::Bracket),
SyntaxKind::LeftBracket => Some(Category::Bracket),
SyntaxKind::RightBracket => Some(Category::Bracket),
SyntaxKind::LeftParen => Some(Category::Bracket),
SyntaxKind::RightParen => Some(Category::Bracket),
SyntaxKind::Comma => Some(Category::Punctuation),
SyntaxKind::Semicolon => Some(Category::Punctuation),
SyntaxKind::Colon => Some(Category::Punctuation),
SyntaxKind::Star => match parent.kind() {
SyntaxKind::Strong => None,
_ => Some(Category::Operator),
},
SyntaxKind::Underscore => match parent.kind() {
SyntaxKind::Script => Some(Category::MathOperator),
_ => None,
},
SyntaxKind::Dollar => Some(Category::MathDelimiter),
SyntaxKind::Plus => Some(match parent.kind() {
SyntaxKind::EnumItem => Category::ListMarker,
_ => Category::Operator,
}),
SyntaxKind::Minus => Some(match parent.kind() {
SyntaxKind::ListItem => Category::ListMarker,
_ => Category::Operator,
}),
SyntaxKind::Slash => Some(match parent.kind() {
SyntaxKind::DescItem => Category::ListMarker,
SyntaxKind::Frac => Category::MathOperator,
_ => Category::Operator,
}),
SyntaxKind::Hat => Some(Category::MathOperator),
SyntaxKind::Amp => Some(Category::MathOperator),
SyntaxKind::Dot => Some(Category::Punctuation),
SyntaxKind::Eq => match parent.kind() {
SyntaxKind::Heading => None,
_ => Some(Category::Operator),
},
SyntaxKind::EqEq => Some(Category::Operator),
SyntaxKind::ExclEq => Some(Category::Operator),
SyntaxKind::Lt => Some(Category::Operator),
SyntaxKind::LtEq => Some(Category::Operator),
SyntaxKind::Gt => Some(Category::Operator),
SyntaxKind::GtEq => Some(Category::Operator),
SyntaxKind::PlusEq => Some(Category::Operator),
SyntaxKind::HyphEq => Some(Category::Operator),
SyntaxKind::StarEq => Some(Category::Operator),
SyntaxKind::SlashEq => Some(Category::Operator),
SyntaxKind::Dots => Some(Category::Operator),
SyntaxKind::Arrow => Some(Category::Operator),
SyntaxKind::Not => Some(Category::Keyword),
SyntaxKind::And => Some(Category::Keyword),
SyntaxKind::Or => Some(Category::Keyword),
SyntaxKind::None => Some(Category::KeywordLiteral),
SyntaxKind::Auto => Some(Category::KeywordLiteral),
SyntaxKind::Let => Some(Category::Keyword),
SyntaxKind::Set => Some(Category::Keyword),
SyntaxKind::Show => Some(Category::Keyword),
SyntaxKind::If => Some(Category::Keyword),
SyntaxKind::Else => Some(Category::Keyword),
SyntaxKind::For => Some(Category::Keyword),
SyntaxKind::In => Some(Category::Keyword),
SyntaxKind::While => Some(Category::Keyword),
SyntaxKind::Break => Some(Category::Keyword),
SyntaxKind::Continue => Some(Category::Keyword),
SyntaxKind::Return => Some(Category::Keyword),
SyntaxKind::Import => Some(Category::Keyword),
SyntaxKind::Include => Some(Category::Keyword),
SyntaxKind::From => Some(Category::Keyword),
SyntaxKind::Markup { .. } => match parent.kind() {
SyntaxKind::DescItem
if parent
.children()
.take_while(|child| child.kind() != &SyntaxKind::Colon)
.find(|c| matches!(c.kind(), SyntaxKind::Markup { .. }))
.map_or(false, |ident| std::ptr::eq(ident, child)) =>
{
Some(Category::ListTerm)
}
_ => None,
},
SyntaxKind::Text(_) => None,
SyntaxKind::Linebreak => Some(Category::Escape),
SyntaxKind::Escape(_) => Some(Category::Escape),
SyntaxKind::Shorthand(_) => Some(Category::Shorthand),
SyntaxKind::Symbol(_) => Some(Category::Symbol),
SyntaxKind::SmartQuote { .. } => Some(Category::SmartQuote),
SyntaxKind::Strong => Some(Category::Strong),
SyntaxKind::Emph => Some(Category::Emph),
SyntaxKind::Raw(_) => Some(Category::Raw),
SyntaxKind::Link(_) => Some(Category::Link),
SyntaxKind::Label(_) => Some(Category::Label),
SyntaxKind::Ref(_) => Some(Category::Ref),
SyntaxKind::Heading => Some(Category::Heading),
SyntaxKind::ListItem => Some(Category::ListItem),
SyntaxKind::EnumItem => Some(Category::ListItem),
SyntaxKind::EnumNumbering(_) => Some(Category::ListMarker),
SyntaxKind::DescItem => Some(Category::ListItem),
SyntaxKind::Math => None,
SyntaxKind::Atom(_) => None,
SyntaxKind::Script => None,
SyntaxKind::Frac => None,
SyntaxKind::AlignPoint => None,
SyntaxKind::Ident(_) => match parent.kind() {
SyntaxKind::Markup { .. }
| SyntaxKind::Math
| SyntaxKind::Script
| SyntaxKind::Frac => Some(Category::Interpolated),
SyntaxKind::FuncCall => Some(Category::Function),
SyntaxKind::MethodCall if i > 0 => Some(Category::Function),
SyntaxKind::Closure if i == 0 => Some(Category::Function),
SyntaxKind::SetRule => Some(Category::Function),
SyntaxKind::ShowRule
if parent
.children()
.rev()
.skip_while(|child| child.kind() != &SyntaxKind::Colon)
.find(|c| matches!(c.kind(), SyntaxKind::Ident(_)))
.map_or(false, |ident| std::ptr::eq(ident, child)) =>
{
Some(Category::Function)
}
_ => None,
},
SyntaxKind::Bool(_) => Some(Category::KeywordLiteral),
SyntaxKind::Int(_) => Some(Category::Number),
SyntaxKind::Float(_) => Some(Category::Number),
SyntaxKind::Numeric(_, _) => Some(Category::Number),
SyntaxKind::Str(_) => Some(Category::String),
SyntaxKind::CodeBlock => None,
SyntaxKind::ContentBlock => None,
SyntaxKind::Parenthesized => None,
SyntaxKind::Array => None,
SyntaxKind::Dict => None,
SyntaxKind::Named => None,
SyntaxKind::Keyed => None,
SyntaxKind::Unary => None,
SyntaxKind::Binary => None,
SyntaxKind::FieldAccess => None,
SyntaxKind::FuncCall => None,
SyntaxKind::MethodCall => None,
SyntaxKind::Args => None,
SyntaxKind::Spread => None,
SyntaxKind::Closure => None,
SyntaxKind::Params => None,
SyntaxKind::LetBinding => None,
SyntaxKind::SetRule => None,
SyntaxKind::ShowRule => None,
SyntaxKind::Conditional => None,
SyntaxKind::WhileLoop => None,
SyntaxKind::ForLoop => None,
SyntaxKind::ForPattern => None,
SyntaxKind::ModuleImport => None,
SyntaxKind::ImportItems => None,
SyntaxKind::ModuleInclude => None,
SyntaxKind::LoopBreak => None,
SyntaxKind::LoopContinue => None,
SyntaxKind::FuncReturn => None,
SyntaxKind::Error(_, _) => Some(Category::Error),
}
}
/// Return the TextMate grammar scope for the given highlighting category.
pub fn tm_scope(&self) -> &'static str {
match self {
Self::Comment => "comment.typst",
Self::Bracket => "punctuation.definition.bracket.typst",
Self::Punctuation => "punctuation.typst",
Self::Escape => "constant.character.escape.typst",
Self::Shorthand => "constant.character.shorthand.typst",
Self::Symbol => "constant.symbol.typst",
Self::SmartQuote => "constant.character.quote.typst",
Self::Strong => "markup.bold.typst",
Self::Emph => "markup.italic.typst",
Self::Link => "markup.underline.link.typst",
Self::Raw => "markup.raw.typst",
Self::MathDelimiter => "punctuation.definition.math.typst",
Self::MathOperator => "keyword.operator.math.typst",
Self::Heading => "markup.heading.typst",
Self::ListItem => "markup.list.typst",
Self::ListMarker => "punctuation.definition.list.typst",
Self::ListTerm => "markup.list.term.typst",
Self::Label => "entity.name.label.typst",
Self::Ref => "markup.other.reference.typst",
Self::Keyword => "keyword.typst",
Self::Operator => "keyword.operator.typst",
Self::KeywordLiteral => "constant.language.typst",
Self::Number => "constant.numeric.typst",
Self::String => "string.quoted.double.typst",
Self::Function => "entity.name.function.typst",
Self::Interpolated => "meta.interpolation.typst",
Self::Error => "invalid.typst",
}
}
}
#[cfg(test)]
mod tests {
use super::super::Source;
use super::*;
#[test]
fn test_highlighting() {
use Category::*;
#[track_caller]
fn test(text: &str, goal: &[(Range<usize>, Category)]) {
let mut vec = vec![];
let source = Source::detached(text);
let full = 0..text.len();
highlight_categories(source.root(), full, &mut |range, category| {
vec.push((range, category));
});
assert_eq!(vec, goal);
}
test("= *AB*", &[(0..6, Heading), (2..6, Strong)]);
test(
"#f(x + 1)",
&[
(0..2, Function),
(2..3, Bracket),
(5..6, Operator),
(7..8, Number),
(8..9, Bracket),
],
);
test(
"#let f(x) = x",
&[
(0..4, Keyword),
(5..6, Function),
(6..7, Bracket),
(8..9, Bracket),
(10..11, Operator),
],
);
}
}

1
src/syntax/mod.rs
View File

@ -1,7 +1,6 @@
//! Syntax definition, parsing, and highlighting.
pub mod ast;
pub mod highlight;
mod incremental;
mod kind;

4
src/syntax/tokens.rs
View File

@ -304,8 +304,10 @@ impl Tokens<'_> {
Some(keyword) => keyword,
None => SyntaxKind::Ident(read.into()),
}
} else {
} else if self.mode == TokenMode::Markup {
self.text(start)
} else {
SyntaxKind::Atom("#".into())
}
}

BIN
tests/ref/math/syntax.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 52 KiB

12
tests/src/benches.rs
View File

@ -22,7 +22,6 @@ main!(
bench_eval,
bench_typeset,
bench_compile,
bench_highlight,
bench_render,
);
@ -63,17 +62,6 @@ fn bench_edit(iai: &mut Iai) {
iai.run(|| black_box(source.edit(1168..1171, "_Uhr_")));
}
fn bench_highlight(iai: &mut Iai) {
let source = Source::detached(TEXT);
iai.run(|| {
typst::syntax::highlight::highlight_categories(
source.root(),
0..source.len_bytes(),
&mut |_, _| {},
)
});
}
fn bench_eval(iai: &mut Iai) {
let world = BenchWorld::new();
let route = typst::model::Route::default();