konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename two syntax types

Browse Source
This commit is contained in:
Laurenz 2022-11-22 14:48:08 +01:00
parent 2ce727fc95
commit b476de87b7
11 changed files with 825 additions and 821 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/model/eval.rs
View File

@ -11,7 +11,7 @@ use super::{
};
use crate::diag::{bail, error, At, SourceResult, StrResult, Trace, Tracepoint};
use crate::geom::{Abs, Angle, Em, Fr, Ratio};
use crate::syntax::ast::TypedNode;
use crate::syntax::ast::AstNode;
use crate::syntax::{ast, SourceId, Span, Spanned, Unit};
use crate::util::{format_eco, EcoString};
use crate::World;

2
src/model/func.rs
View File

@ -8,7 +8,7 @@ use super::{
Args, Eval, Flow, Node, NodeId, Route, Scope, Scopes, Selector, StyleMap, Value, Vm,
};
use crate::diag::{bail, SourceResult, StrResult};
use crate::syntax::ast::{self, Expr, TypedNode};
use crate::syntax::ast::{self, AstNode, Expr};
use crate::syntax::{SourceId, Span, SyntaxNode};
use crate::util::EcoString;
use crate::World;

263
src/syntax/ast.rs
View File

@ -5,11 +5,11 @@
use std::num::NonZeroUsize;
use std::ops::Deref;
use super::{NodeKind, RawFields, Span, SyntaxNode, Unit};
use super::{RawFields, Span, SyntaxKind, SyntaxNode, Unit};
use crate::util::EcoString;
/// A typed AST node.
pub trait TypedNode: Sized {
pub trait AstNode: Sized {
/// Convert a node into its typed variant.
fn from_untyped(node: &SyntaxNode) -> Option<Self>;
@ -24,7 +24,7 @@ pub trait TypedNode: Sized {
macro_rules! node {
($(#[$attr:meta])* $name:ident) => {
node!{ $(#[$attr])* $name: NodeKind::$name { .. } }
node!{ $(#[$attr])* $name: SyntaxKind::$name { .. } }
};
($(#[$attr:meta])* $name:ident: $variants:pat) => {
#[derive(Debug, Clone, PartialEq, Hash)]
@ -32,7 +32,7 @@ macro_rules! node {
$(#[$attr])*
pub struct $name(SyntaxNode);
impl TypedNode for $name {
impl AstNode for $name {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
if matches!(node.kind(), $variants) {
Some(Self(node.clone()))
@ -62,7 +62,8 @@ impl Markup {
.filter(move |node| {
// Ignore linebreak directly after statements without semicolons.
let kind = node.kind();
let keep = !was_stmt || !matches!(kind, NodeKind::Space { newlines: 1 });
let keep =
!was_stmt || !matches!(kind, SyntaxKind::Space { newlines: 1 });
was_stmt = kind.is_stmt();
keep
})
@ -112,26 +113,26 @@ pub enum MarkupNode {
Expr(Expr),
}
impl TypedNode for MarkupNode {
impl AstNode for MarkupNode {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Space { .. } => node.cast().map(Self::Space),
NodeKind::Linebreak => node.cast().map(Self::Linebreak),
NodeKind::Text(_) => node.cast().map(Self::Text),
NodeKind::Escape(_) => node.cast().map(Self::Escape),
NodeKind::Shorthand(_) => node.cast().map(Self::Shorthand),
NodeKind::SmartQuote { .. } => node.cast().map(Self::SmartQuote),
NodeKind::Strong => node.cast().map(Self::Strong),
NodeKind::Emph => node.cast().map(Self::Emph),
NodeKind::Raw(_) => node.cast().map(Self::Raw),
NodeKind::Link(_) => node.cast().map(Self::Link),
NodeKind::Label(_) => node.cast().map(Self::Label),
NodeKind::Ref(_) => node.cast().map(Self::Ref),
NodeKind::Heading => node.cast().map(Self::Heading),
NodeKind::ListItem => node.cast().map(Self::List),
NodeKind::EnumItem => node.cast().map(Self::Enum),
NodeKind::DescItem => node.cast().map(Self::Desc),
NodeKind::Math => node.cast().map(Self::Math),
SyntaxKind::Space { .. } => node.cast().map(Self::Space),
SyntaxKind::Linebreak => node.cast().map(Self::Linebreak),
SyntaxKind::Text(_) => node.cast().map(Self::Text),
SyntaxKind::Escape(_) => node.cast().map(Self::Escape),
SyntaxKind::Shorthand(_) => node.cast().map(Self::Shorthand),
SyntaxKind::SmartQuote { .. } => node.cast().map(Self::SmartQuote),
SyntaxKind::Strong => node.cast().map(Self::Strong),
SyntaxKind::Emph => node.cast().map(Self::Emph),
SyntaxKind::Raw(_) => node.cast().map(Self::Raw),
SyntaxKind::Link(_) => node.cast().map(Self::Link),
SyntaxKind::Label(_) => node.cast().map(Self::Label),
SyntaxKind::Ref(_) => node.cast().map(Self::Ref),
SyntaxKind::Heading => node.cast().map(Self::Heading),
SyntaxKind::ListItem => node.cast().map(Self::List),
SyntaxKind::EnumItem => node.cast().map(Self::Enum),
SyntaxKind::DescItem => node.cast().map(Self::Desc),
SyntaxKind::Math => node.cast().map(Self::Math),
_ => node.cast().map(Self::Expr),
}
}
@ -169,7 +170,7 @@ impl Space {
/// Get the number of newlines.
pub fn newlines(&self) -> usize {
match self.0.kind() {
&NodeKind::Space { newlines } => newlines,
&SyntaxKind::Space { newlines } => newlines,
_ => panic!("space is of wrong kind"),
}
}
@ -189,7 +190,7 @@ impl Text {
/// Get the text.
pub fn get(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Text(v) => v,
SyntaxKind::Text(v) => v,
_ => panic!("text is of wrong kind"),
}
}
@ -204,7 +205,7 @@ impl Escape {
/// Get the escaped character.
pub fn get(&self) -> char {
match self.0.kind() {
&NodeKind::Escape(v) => v,
&SyntaxKind::Escape(v) => v,
_ => panic!("escape is of wrong kind"),
}
}
@ -220,7 +221,7 @@ impl Shorthand {
/// Get the shorthanded character.
pub fn get(&self) -> char {
match self.0.kind() {
&NodeKind::Shorthand(v) => v,
&SyntaxKind::Shorthand(v) => v,
_ => panic!("shorthand is of wrong kind"),
}
}
@ -235,7 +236,7 @@ impl SmartQuote {
/// Whether this is a double quote.
pub fn double(&self) -> bool {
match self.0.kind() {
&NodeKind::SmartQuote { double } => double,
&SyntaxKind::SmartQuote { double } => double,
_ => panic!("smart quote is of wrong kind"),
}
}
@ -291,7 +292,7 @@ impl Raw {
/// The raw fields.
fn get(&self) -> &RawFields {
match self.0.kind() {
NodeKind::Raw(v) => v.as_ref(),
SyntaxKind::Raw(v) => v.as_ref(),
_ => panic!("raw is of wrong kind"),
}
}
@ -306,7 +307,7 @@ impl Link {
/// Get the URL.
pub fn url(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Link(url) => url,
SyntaxKind::Link(url) => url,
_ => panic!("link is of wrong kind"),
}
}
@ -321,7 +322,7 @@ impl Label {
/// Get the label.
pub fn get(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Label(v) => v,
SyntaxKind::Label(v) => v,
_ => panic!("label is of wrong kind"),
}
}
@ -336,7 +337,7 @@ impl Ref {
/// Get the target.
pub fn get(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Ref(v) => v,
SyntaxKind::Ref(v) => v,
_ => panic!("reference is of wrong kind"),
}
}
@ -357,7 +358,7 @@ impl Heading {
pub fn level(&self) -> NonZeroUsize {
self.0
.children()
.filter(|n| n.kind() == &NodeKind::Eq)
.filter(|n| n.kind() == &SyntaxKind::Eq)
.count()
.try_into()
.expect("heading is missing equals sign")
@ -385,7 +386,7 @@ impl EnumItem {
/// The explicit numbering, if any: `23.`.
pub fn number(&self) -> Option<usize> {
self.0.children().find_map(|node| match node.kind() {
NodeKind::EnumNumbering(num) => Some(*num),
SyntaxKind::EnumNumbering(num) => Some(*num),
_ => None,
})
}
@ -458,17 +459,17 @@ pub enum MathNode {
Expr(Expr),
}
impl TypedNode for MathNode {
impl AstNode for MathNode {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Space { .. } => node.cast().map(Self::Space),
NodeKind::Linebreak => node.cast().map(Self::Linebreak),
NodeKind::Escape(_) => node.cast().map(Self::Escape),
NodeKind::Atom(_) => node.cast().map(Self::Atom),
NodeKind::Script => node.cast().map(Self::Script),
NodeKind::Frac => node.cast().map(Self::Frac),
NodeKind::Align => node.cast().map(Self::Align),
NodeKind::Math => node.cast().map(Self::Group),
SyntaxKind::Space { .. } => node.cast().map(Self::Space),
SyntaxKind::Linebreak => node.cast().map(Self::Linebreak),
SyntaxKind::Escape(_) => node.cast().map(Self::Escape),
SyntaxKind::Atom(_) => node.cast().map(Self::Atom),
SyntaxKind::Script => node.cast().map(Self::Script),
SyntaxKind::Frac => node.cast().map(Self::Frac),
SyntaxKind::Align => node.cast().map(Self::Align),
SyntaxKind::Math => node.cast().map(Self::Group),
_ => node.cast().map(Self::Expr),
}
}
@ -497,7 +498,7 @@ impl Atom {
/// Get the atom's text.
pub fn get(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Atom(v) => v,
SyntaxKind::Atom(v) => v,
_ => panic!("atom is of wrong kind"),
}
}
@ -518,7 +519,7 @@ impl Script {
pub fn sub(&self) -> Option<MathNode> {
self.0
.children()
.skip_while(|node| !matches!(node.kind(), NodeKind::Underscore))
.skip_while(|node| !matches!(node.kind(), SyntaxKind::Underscore))
.nth(1)
.map(|node| node.cast().expect("script node has invalid subscript"))
}
@ -527,7 +528,7 @@ impl Script {
pub fn sup(&self) -> Option<MathNode> {
self.0
.children()
.skip_while(|node| !matches!(node.kind(), NodeKind::Hat))
.skip_while(|node| !matches!(node.kind(), SyntaxKind::Hat))
.nth(1)
.map(|node| node.cast().expect("script node has invalid superscript"))
}
@ -558,7 +559,7 @@ node! {
impl Align {
/// The number of ampersands.
pub fn count(&self) -> usize {
self.0.children().filter(|n| n.kind() == &NodeKind::Amp).count()
self.0.children().filter(|n| n.kind() == &SyntaxKind::Amp).count()
}
}
@ -615,32 +616,32 @@ pub enum Expr {
Return(FuncReturn),
}
impl TypedNode for Expr {
impl AstNode for Expr {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Ident(_) => node.cast().map(Self::Ident),
NodeKind::CodeBlock => node.cast().map(Self::Code),
NodeKind::ContentBlock => node.cast().map(Self::Content),
NodeKind::Parenthesized => node.cast().map(Self::Parenthesized),
NodeKind::Array => node.cast().map(Self::Array),
NodeKind::Dict => node.cast().map(Self::Dict),
NodeKind::Unary => node.cast().map(Self::Unary),
NodeKind::Binary => node.cast().map(Self::Binary),
NodeKind::FieldAccess => node.cast().map(Self::FieldAccess),
NodeKind::FuncCall => node.cast().map(Self::FuncCall),
NodeKind::MethodCall => node.cast().map(Self::MethodCall),
NodeKind::Closure => node.cast().map(Self::Closure),
NodeKind::LetBinding => node.cast().map(Self::Let),
NodeKind::SetRule => node.cast().map(Self::Set),
NodeKind::ShowRule => node.cast().map(Self::Show),
NodeKind::Conditional => node.cast().map(Self::Conditional),
NodeKind::WhileLoop => node.cast().map(Self::While),
NodeKind::ForLoop => node.cast().map(Self::For),
NodeKind::ModuleImport => node.cast().map(Self::Import),
NodeKind::ModuleInclude => node.cast().map(Self::Include),
NodeKind::LoopBreak => node.cast().map(Self::Break),
NodeKind::LoopContinue => node.cast().map(Self::Continue),
NodeKind::FuncReturn => node.cast().map(Self::Return),
SyntaxKind::Ident(_) => node.cast().map(Self::Ident),
SyntaxKind::CodeBlock => node.cast().map(Self::Code),
SyntaxKind::ContentBlock => node.cast().map(Self::Content),
SyntaxKind::Parenthesized => node.cast().map(Self::Parenthesized),
SyntaxKind::Array => node.cast().map(Self::Array),
SyntaxKind::Dict => node.cast().map(Self::Dict),
SyntaxKind::Unary => node.cast().map(Self::Unary),
SyntaxKind::Binary => node.cast().map(Self::Binary),
SyntaxKind::FieldAccess => node.cast().map(Self::FieldAccess),
SyntaxKind::FuncCall => node.cast().map(Self::FuncCall),
SyntaxKind::MethodCall => node.cast().map(Self::MethodCall),
SyntaxKind::Closure => node.cast().map(Self::Closure),
SyntaxKind::LetBinding => node.cast().map(Self::Let),
SyntaxKind::SetRule => node.cast().map(Self::Set),
SyntaxKind::ShowRule => node.cast().map(Self::Show),
SyntaxKind::Conditional => node.cast().map(Self::Conditional),
SyntaxKind::WhileLoop => node.cast().map(Self::While),
SyntaxKind::ForLoop => node.cast().map(Self::For),
SyntaxKind::ModuleImport => node.cast().map(Self::Import),
SyntaxKind::ModuleInclude => node.cast().map(Self::Include),
SyntaxKind::LoopBreak => node.cast().map(Self::Break),
SyntaxKind::LoopContinue => node.cast().map(Self::Continue),
SyntaxKind::FuncReturn => node.cast().map(Self::Return),
_ => node.cast().map(Self::Lit),
}
}
@ -696,26 +697,26 @@ impl Expr {
node! {
/// A literal: `1`, `true`, ...
Lit: NodeKind::None
| NodeKind::Auto
| NodeKind::Bool(_)
| NodeKind::Int(_)
| NodeKind::Float(_)
| NodeKind::Numeric(_, _)
| NodeKind::Str(_)
Lit: SyntaxKind::None
| SyntaxKind::Auto
| SyntaxKind::Bool(_)
| SyntaxKind::Int(_)
| SyntaxKind::Float(_)
| SyntaxKind::Numeric(_, _)
| SyntaxKind::Str(_)
}
impl Lit {
/// The kind of literal.
pub fn kind(&self) -> LitKind {
match *self.0.kind() {
NodeKind::None => LitKind::None,
NodeKind::Auto => LitKind::Auto,
NodeKind::Bool(v) => LitKind::Bool(v),
NodeKind::Int(v) => LitKind::Int(v),
NodeKind::Float(v) => LitKind::Float(v),
NodeKind::Numeric(v, unit) => LitKind::Numeric(v, unit),
NodeKind::Str(ref v) => LitKind::Str(v.clone()),
SyntaxKind::None => LitKind::None,
SyntaxKind::Auto => LitKind::Auto,
SyntaxKind::Bool(v) => LitKind::Bool(v),
SyntaxKind::Int(v) => LitKind::Int(v),
SyntaxKind::Float(v) => LitKind::Float(v),
SyntaxKind::Numeric(v, unit) => LitKind::Numeric(v, unit),
SyntaxKind::Str(ref v) => LitKind::Str(v.clone()),
_ => panic!("literal is of wrong kind"),
}
}
@ -799,10 +800,10 @@ pub enum ArrayItem {
Spread(Expr),
}
impl TypedNode for ArrayItem {
impl AstNode for ArrayItem {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Spread => node.cast_first_child().map(Self::Spread),
SyntaxKind::Spread => node.cast_first_child().map(Self::Spread),
_ => node.cast().map(Self::Pos),
}
}
@ -838,12 +839,12 @@ pub enum DictItem {
Spread(Expr),
}
impl TypedNode for DictItem {
impl AstNode for DictItem {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Named => node.cast().map(Self::Named),
NodeKind::Keyed => node.cast().map(Self::Keyed),
NodeKind::Spread => node.cast_first_child().map(Self::Spread),
SyntaxKind::Named => node.cast().map(Self::Named),
SyntaxKind::Keyed => node.cast().map(Self::Keyed),
SyntaxKind::Spread => node.cast_first_child().map(Self::Spread),
_ => None,
}
}
@ -885,7 +886,7 @@ impl Keyed {
self.0
.children()
.find_map(|node| match node.kind() {
NodeKind::Str(key) => Some(key.clone()),
SyntaxKind::Str(key) => Some(key.clone()),
_ => None,
})
.expect("keyed pair is missing key")
@ -930,11 +931,11 @@ pub enum UnOp {
impl UnOp {
/// Try to convert the token into a unary operation.
pub fn from_token(token: &NodeKind) -> Option<Self> {
pub fn from_token(token: &SyntaxKind) -> Option<Self> {
Some(match token {
NodeKind::Plus => Self::Pos,
NodeKind::Minus => Self::Neg,
NodeKind::Not => Self::Not,
SyntaxKind::Plus => Self::Pos,
SyntaxKind::Minus => Self::Neg,
SyntaxKind::Not => Self::Not,
_ => return None,
})
}
@ -969,11 +970,11 @@ impl Binary {
self.0
.children()
.find_map(|node| match node.kind() {
NodeKind::Not => {
SyntaxKind::Not => {
not = true;
None
}
NodeKind::In if not => Some(BinOp::NotIn),
SyntaxKind::In if not => Some(BinOp::NotIn),
_ => BinOp::from_token(node.kind()),
})
.expect("binary operation is missing operator")
@ -1039,26 +1040,26 @@ pub enum BinOp {
impl BinOp {
/// Try to convert the token into a binary operation.
pub fn from_token(token: &NodeKind) -> Option<Self> {
pub fn from_token(token: &SyntaxKind) -> Option<Self> {
Some(match token {
NodeKind::Plus => Self::Add,
NodeKind::Minus => Self::Sub,
NodeKind::Star => Self::Mul,
NodeKind::Slash => Self::Div,
NodeKind::And => Self::And,
NodeKind::Or => Self::Or,
NodeKind::EqEq => Self::Eq,
NodeKind::ExclEq => Self::Neq,
NodeKind::Lt => Self::Lt,
NodeKind::LtEq => Self::Leq,
NodeKind::Gt => Self::Gt,
NodeKind::GtEq => Self::Geq,
NodeKind::Eq => Self::Assign,
NodeKind::In => Self::In,
NodeKind::PlusEq => Self::AddAssign,
NodeKind::HyphEq => Self::SubAssign,
NodeKind::StarEq => Self::MulAssign,
NodeKind::SlashEq => Self::DivAssign,
SyntaxKind::Plus => Self::Add,
SyntaxKind::Minus => Self::Sub,
SyntaxKind::Star => Self::Mul,
SyntaxKind::Slash => Self::Div,
SyntaxKind::And => Self::And,
SyntaxKind::Or => Self::Or,
SyntaxKind::EqEq => Self::Eq,
SyntaxKind::ExclEq => Self::Neq,
SyntaxKind::Lt => Self::Lt,
SyntaxKind::LtEq => Self::Leq,
SyntaxKind::Gt => Self::Gt,
SyntaxKind::GtEq => Self::Geq,
SyntaxKind::Eq => Self::Assign,
SyntaxKind::In => Self::In,
SyntaxKind::PlusEq => Self::AddAssign,
SyntaxKind::HyphEq => Self::SubAssign,
SyntaxKind::StarEq => Self::MulAssign,
SyntaxKind::SlashEq => Self::DivAssign,
_ => return None,
})
}
@ -1231,11 +1232,11 @@ pub enum Arg {
Spread(Expr),
}
impl TypedNode for Arg {
impl AstNode for Arg {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Named => node.cast().map(Self::Named),
NodeKind::Spread => node.cast_first_child().map(Self::Spread),
SyntaxKind::Named => node.cast().map(Self::Named),
SyntaxKind::Spread => node.cast_first_child().map(Self::Spread),
_ => node.cast().map(Self::Pos),
}
}
@ -1266,7 +1267,7 @@ impl Closure {
pub fn params(&self) -> impl DoubleEndedIterator<Item = Param> + '_ {
self.0
.children()
.find(|x| x.kind() == &NodeKind::Params)
.find(|x| x.kind() == &SyntaxKind::Params)
.expect("closure is missing parameter list")
.children()
.filter_map(SyntaxNode::cast)
@ -1289,12 +1290,12 @@ pub enum Param {
Sink(Ident),
}
impl TypedNode for Param {
impl AstNode for Param {
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
match node.kind() {
NodeKind::Ident(_) => node.cast().map(Self::Pos),
NodeKind::Named => node.cast().map(Self::Named),
NodeKind::Spread => node.cast_first_child().map(Self::Sink),
SyntaxKind::Ident(_) => node.cast().map(Self::Pos),
SyntaxKind::Named => node.cast().map(Self::Named),
SyntaxKind::Spread => node.cast_first_child().map(Self::Sink),
_ => None,
}
}
@ -1357,7 +1358,7 @@ impl SetRule {
pub fn condition(&self) -> Option<Expr> {
self.0
.children()
.skip_while(|child| child.kind() != &NodeKind::If)
.skip_while(|child| child.kind() != &SyntaxKind::If)
.find_map(SyntaxNode::cast)
}
}
@ -1373,7 +1374,7 @@ impl ShowRule {
self.0
.children()
.rev()
.skip_while(|child| child.kind() != &NodeKind::Colon)
.skip_while(|child| child.kind() != &SyntaxKind::Colon)
.find_map(SyntaxNode::cast)
}
@ -1482,8 +1483,8 @@ impl ModuleImport {
self.0
.children()
.find_map(|node| match node.kind() {
NodeKind::Star => Some(Imports::Wildcard),
NodeKind::ImportItems => {
SyntaxKind::Star => Some(Imports::Wildcard),
SyntaxKind::ImportItems => {
let items = node.children().filter_map(SyntaxNode::cast).collect();
Some(Imports::Items(items))
}
@ -1550,7 +1551,7 @@ impl Ident {
/// Get the identifier.
pub fn get(&self) -> &EcoString {
match self.0.kind() {
NodeKind::Ident(id) => id,
SyntaxKind::Ident(id) => id,
_ => panic!("identifier is of wrong kind"),
}
}
@ -1558,7 +1559,7 @@ impl Ident {
/// Take out the container identifier.
pub fn take(self) -> EcoString {
match self.0.take() {
NodeKind::Ident(id) => id,
SyntaxKind::Ident(id) => id,
_ => panic!("identifier is of wrong kind"),
}
}

262
src/syntax/highlight.rs
View File

@ -6,7 +6,7 @@ use std::ops::Range;
use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme};
use syntect::parsing::Scope;
use super::{parse, NodeKind, SyntaxNode};
use super::{parse, SyntaxKind, SyntaxNode};
/// Highlight source text into a standalone HTML document.
pub fn highlight_html(text: &str, theme: &Theme) -> String {
@ -194,170 +194,170 @@ impl Category {
i: usize,
) -> Option<Category> {
match child.kind() {
NodeKind::LineComment => Some(Category::Comment),
NodeKind::BlockComment => Some(Category::Comment),
NodeKind::Space { .. } => None,
SyntaxKind::LineComment => Some(Category::Comment),
SyntaxKind::BlockComment => Some(Category::Comment),
SyntaxKind::Space { .. } => None,
NodeKind::LeftBrace => Some(Category::Bracket),
NodeKind::RightBrace => Some(Category::Bracket),
NodeKind::LeftBracket => Some(Category::Bracket),
NodeKind::RightBracket => Some(Category::Bracket),
NodeKind::LeftParen => Some(Category::Bracket),
NodeKind::RightParen => Some(Category::Bracket),
NodeKind::Comma => Some(Category::Punctuation),
NodeKind::Semicolon => Some(Category::Punctuation),
NodeKind::Colon => Some(Category::Punctuation),
NodeKind::Star => match parent.kind() {
NodeKind::Strong => None,
SyntaxKind::LeftBrace => Some(Category::Bracket),
SyntaxKind::RightBrace => Some(Category::Bracket),
SyntaxKind::LeftBracket => Some(Category::Bracket),
SyntaxKind::RightBracket => Some(Category::Bracket),
SyntaxKind::LeftParen => Some(Category::Bracket),
SyntaxKind::RightParen => Some(Category::Bracket),
SyntaxKind::Comma => Some(Category::Punctuation),
SyntaxKind::Semicolon => Some(Category::Punctuation),
SyntaxKind::Colon => Some(Category::Punctuation),
SyntaxKind::Star => match parent.kind() {
SyntaxKind::Strong => None,
_ => Some(Category::Operator),
},
NodeKind::Underscore => match parent.kind() {
NodeKind::Script => Some(Category::MathOperator),
SyntaxKind::Underscore => match parent.kind() {
SyntaxKind::Script => Some(Category::MathOperator),
_ => None,
},
NodeKind::Dollar => Some(Category::MathDelimiter),
NodeKind::Plus => Some(match parent.kind() {
NodeKind::EnumItem => Category::ListMarker,
SyntaxKind::Dollar => Some(Category::MathDelimiter),
SyntaxKind::Plus => Some(match parent.kind() {
SyntaxKind::EnumItem => Category::ListMarker,
_ => Category::Operator,
}),
NodeKind::Minus => Some(match parent.kind() {
NodeKind::ListItem => Category::ListMarker,
SyntaxKind::Minus => Some(match parent.kind() {
SyntaxKind::ListItem => Category::ListMarker,
_ => Category::Operator,
}),
NodeKind::Slash => Some(match parent.kind() {
NodeKind::DescItem => Category::ListMarker,
NodeKind::Frac => Category::MathOperator,
SyntaxKind::Slash => Some(match parent.kind() {
SyntaxKind::DescItem => Category::ListMarker,
SyntaxKind::Frac => Category::MathOperator,
_ => Category::Operator,
}),
NodeKind::Hat => Some(Category::MathOperator),
NodeKind::Amp => Some(Category::MathOperator),
NodeKind::Dot => Some(Category::Punctuation),
NodeKind::Eq => match parent.kind() {
NodeKind::Heading => None,
SyntaxKind::Hat => Some(Category::MathOperator),
SyntaxKind::Amp => Some(Category::MathOperator),
SyntaxKind::Dot => Some(Category::Punctuation),
SyntaxKind::Eq => match parent.kind() {
SyntaxKind::Heading => None,
_ => Some(Category::Operator),
},
NodeKind::EqEq => Some(Category::Operator),
NodeKind::ExclEq => Some(Category::Operator),
NodeKind::Lt => Some(Category::Operator),
NodeKind::LtEq => Some(Category::Operator),
NodeKind::Gt => Some(Category::Operator),
NodeKind::GtEq => Some(Category::Operator),
NodeKind::PlusEq => Some(Category::Operator),
NodeKind::HyphEq => Some(Category::Operator),
NodeKind::StarEq => Some(Category::Operator),
NodeKind::SlashEq => Some(Category::Operator),
NodeKind::Dots => Some(Category::Operator),
NodeKind::Arrow => Some(Category::Operator),
SyntaxKind::EqEq => Some(Category::Operator),
SyntaxKind::ExclEq => Some(Category::Operator),
SyntaxKind::Lt => Some(Category::Operator),
SyntaxKind::LtEq => Some(Category::Operator),
SyntaxKind::Gt => Some(Category::Operator),
SyntaxKind::GtEq => Some(Category::Operator),
SyntaxKind::PlusEq => Some(Category::Operator),
SyntaxKind::HyphEq => Some(Category::Operator),
SyntaxKind::StarEq => Some(Category::Operator),
SyntaxKind::SlashEq => Some(Category::Operator),
SyntaxKind::Dots => Some(Category::Operator),
SyntaxKind::Arrow => Some(Category::Operator),
NodeKind::Not => Some(Category::Keyword),
NodeKind::And => Some(Category::Keyword),
NodeKind::Or => Some(Category::Keyword),
NodeKind::None => Some(Category::KeywordLiteral),
NodeKind::Auto => Some(Category::KeywordLiteral),
NodeKind::Let => Some(Category::Keyword),
NodeKind::Set => Some(Category::Keyword),
NodeKind::Show => Some(Category::Keyword),
NodeKind::If => Some(Category::Keyword),
NodeKind::Else => Some(Category::Keyword),
NodeKind::For => Some(Category::Keyword),
NodeKind::In => Some(Category::Keyword),
NodeKind::While => Some(Category::Keyword),
NodeKind::Break => Some(Category::Keyword),
NodeKind::Continue => Some(Category::Keyword),
NodeKind::Return => Some(Category::Keyword),
NodeKind::Import => Some(Category::Keyword),
NodeKind::Include => Some(Category::Keyword),
NodeKind::From => Some(Category::Keyword),
SyntaxKind::Not => Some(Category::Keyword),
SyntaxKind::And => Some(Category::Keyword),
SyntaxKind::Or => Some(Category::Keyword),
SyntaxKind::None => Some(Category::KeywordLiteral),
SyntaxKind::Auto => Some(Category::KeywordLiteral),
SyntaxKind::Let => Some(Category::Keyword),
SyntaxKind::Set => Some(Category::Keyword),
SyntaxKind::Show => Some(Category::Keyword),
SyntaxKind::If => Some(Category::Keyword),
SyntaxKind::Else => Some(Category::Keyword),
SyntaxKind::For => Some(Category::Keyword),
SyntaxKind::In => Some(Category::Keyword),
SyntaxKind::While => Some(Category::Keyword),
SyntaxKind::Break => Some(Category::Keyword),
SyntaxKind::Continue => Some(Category::Keyword),
SyntaxKind::Return => Some(Category::Keyword),
SyntaxKind::Import => Some(Category::Keyword),
SyntaxKind::Include => Some(Category::Keyword),
SyntaxKind::From => Some(Category::Keyword),
NodeKind::Markup { .. } => match parent.kind() {
NodeKind::DescItem
SyntaxKind::Markup { .. } => match parent.kind() {
SyntaxKind::DescItem
if parent
.children()
.take_while(|child| child.kind() != &NodeKind::Colon)
.find(|c| matches!(c.kind(), NodeKind::Markup { .. }))
.take_while(|child| child.kind() != &SyntaxKind::Colon)
.find(|c| matches!(c.kind(), SyntaxKind::Markup { .. }))
.map_or(false, |ident| std::ptr::eq(ident, child)) =>
{
Some(Category::ListTerm)
}
_ => None,
},
NodeKind::Text(_) => None,
NodeKind::Linebreak => Some(Category::Escape),
NodeKind::Escape(_) => Some(Category::Escape),
NodeKind::Shorthand(_) => Some(Category::Shorthand),
NodeKind::SmartQuote { .. } => Some(Category::SmartQuote),
NodeKind::Strong => Some(Category::Strong),
NodeKind::Emph => Some(Category::Emph),
NodeKind::Raw(_) => Some(Category::Raw),
NodeKind::Link(_) => Some(Category::Link),
NodeKind::Label(_) => Some(Category::Label),
NodeKind::Ref(_) => Some(Category::Ref),
NodeKind::Heading => Some(Category::Heading),
NodeKind::ListItem => Some(Category::ListItem),
NodeKind::EnumItem => Some(Category::ListItem),
NodeKind::EnumNumbering(_) => Some(Category::ListMarker),
NodeKind::DescItem => Some(Category::ListItem),
NodeKind::Math => Some(Category::Math),
NodeKind::Atom(_) => None,
NodeKind::Script => None,
NodeKind::Frac => None,
NodeKind::Align => None,
SyntaxKind::Text(_) => None,
SyntaxKind::Linebreak => Some(Category::Escape),
SyntaxKind::Escape(_) => Some(Category::Escape),
SyntaxKind::Shorthand(_) => Some(Category::Shorthand),
SyntaxKind::SmartQuote { .. } => Some(Category::SmartQuote),
SyntaxKind::Strong => Some(Category::Strong),
SyntaxKind::Emph => Some(Category::Emph),
SyntaxKind::Raw(_) => Some(Category::Raw),
SyntaxKind::Link(_) => Some(Category::Link),
SyntaxKind::Label(_) => Some(Category::Label),
SyntaxKind::Ref(_) => Some(Category::Ref),
SyntaxKind::Heading => Some(Category::Heading),
SyntaxKind::ListItem => Some(Category::ListItem),
SyntaxKind::EnumItem => Some(Category::ListItem),
SyntaxKind::EnumNumbering(_) => Some(Category::ListMarker),
SyntaxKind::DescItem => Some(Category::ListItem),
SyntaxKind::Math => Some(Category::Math),
SyntaxKind::Atom(_) => None,
SyntaxKind::Script => None,
SyntaxKind::Frac => None,
SyntaxKind::Align => None,
NodeKind::Ident(_) => match parent.kind() {
NodeKind::Markup { .. } => Some(Category::Interpolated),
NodeKind::Math => Some(Category::Interpolated),
NodeKind::FuncCall => Some(Category::Function),
NodeKind::MethodCall if i > 0 => Some(Category::Function),
NodeKind::Closure if i == 0 => Some(Category::Function),
NodeKind::SetRule => Some(Category::Function),
NodeKind::ShowRule
SyntaxKind::Ident(_) => match parent.kind() {
SyntaxKind::Markup { .. } => Some(Category::Interpolated),
SyntaxKind::Math => Some(Category::Interpolated),
SyntaxKind::FuncCall => Some(Category::Function),
SyntaxKind::MethodCall if i > 0 => Some(Category::Function),
SyntaxKind::Closure if i == 0 => Some(Category::Function),
SyntaxKind::SetRule => Some(Category::Function),
SyntaxKind::ShowRule
if parent
.children()
.rev()
.skip_while(|child| child.kind() != &NodeKind::Colon)
.find(|c| matches!(c.kind(), NodeKind::Ident(_)))
.skip_while(|child| child.kind() != &SyntaxKind::Colon)
.find(|c| matches!(c.kind(), SyntaxKind::Ident(_)))
.map_or(false, |ident| std::ptr::eq(ident, child)) =>
{
Some(Category::Function)
}
_ => None,
},
NodeKind::Bool(_) => Some(Category::KeywordLiteral),
NodeKind::Int(_) => Some(Category::Number),
NodeKind::Float(_) => Some(Category::Number),
NodeKind::Numeric(_, _) => Some(Category::Number),
NodeKind::Str(_) => Some(Category::String),
NodeKind::CodeBlock => None,
NodeKind::ContentBlock => None,
NodeKind::Parenthesized => None,
NodeKind::Array => None,
NodeKind::Dict => None,
NodeKind::Named => None,
NodeKind::Keyed => None,
NodeKind::Unary => None,
NodeKind::Binary => None,
NodeKind::FieldAccess => None,
NodeKind::FuncCall => None,
NodeKind::MethodCall => None,
NodeKind::Args => None,
NodeKind::Spread => None,
NodeKind::Closure => None,
NodeKind::Params => None,
NodeKind::LetBinding => None,
NodeKind::SetRule => None,
NodeKind::ShowRule => None,
NodeKind::Conditional => None,
NodeKind::WhileLoop => None,
NodeKind::ForLoop => None,
NodeKind::ForPattern => None,
NodeKind::ModuleImport => None,
NodeKind::ImportItems => None,
NodeKind::ModuleInclude => None,
NodeKind::LoopBreak => None,
NodeKind::LoopContinue => None,
NodeKind::FuncReturn => None,
SyntaxKind::Bool(_) => Some(Category::KeywordLiteral),
SyntaxKind::Int(_) => Some(Category::Number),
SyntaxKind::Float(_) => Some(Category::Number),
SyntaxKind::Numeric(_, _) => Some(Category::Number),
SyntaxKind::Str(_) => Some(Category::String),
SyntaxKind::CodeBlock => None,
SyntaxKind::ContentBlock => None,
SyntaxKind::Parenthesized => None,
SyntaxKind::Array => None,
SyntaxKind::Dict => None,
SyntaxKind::Named => None,
SyntaxKind::Keyed => None,
SyntaxKind::Unary => None,
SyntaxKind::Binary => None,
SyntaxKind::FieldAccess => None,
SyntaxKind::FuncCall => None,
SyntaxKind::MethodCall => None,
SyntaxKind::Args => None,
SyntaxKind::Spread => None,
SyntaxKind::Closure => None,
SyntaxKind::Params => None,
SyntaxKind::LetBinding => None,
SyntaxKind::SetRule => None,
SyntaxKind::ShowRule => None,
SyntaxKind::Conditional => None,
SyntaxKind::WhileLoop => None,
SyntaxKind::ForLoop => None,
SyntaxKind::ForPattern => None,
SyntaxKind::ModuleImport => None,
SyntaxKind::ImportItems => None,
SyntaxKind::ModuleInclude => None,
SyntaxKind::LoopBreak => None,
SyntaxKind::LoopContinue => None,
SyntaxKind::FuncReturn => None,
NodeKind::Error(_, _) => Some(Category::Error),
SyntaxKind::Error(_, _) => Some(Category::Error),
}
}

42
src/syntax/incremental.rs
View File

@ -2,7 +2,7 @@ use std::ops::Range;
use super::{
is_newline, parse, reparse_code_block, reparse_content_block,
reparse_markup_elements, NodeKind, Span, SyntaxNode,
reparse_markup_elements, Span, SyntaxKind, SyntaxNode,
};
/// Refresh the given syntax node with as little parsing as possible.
@ -36,7 +36,7 @@ fn try_reparse(
outermost: bool,
safe_to_replace: bool,
) -> Option<Range<usize>> {
let is_markup = matches!(node.kind(), NodeKind::Markup { .. });
let is_markup = matches!(node.kind(), SyntaxKind::Markup { .. });
let original_count = node.children().len();
let original_offset = offset;
@ -78,7 +78,7 @@ fn try_reparse(
} else {
// Update compulsary state of `ahead_nontrivia`.
if let Some(ahead_nontrivia) = ahead.as_mut() {
if let NodeKind::Space { newlines: (1..) } = child.kind() {
if let SyntaxKind::Space { newlines: (1..) } = child.kind() {
ahead_nontrivia.newline();
}
}
@ -87,8 +87,8 @@ fn try_reparse(
// reject text that points to the special case for URL
// evasion and line comments.
if !child.kind().is_space()
&& child.kind() != &NodeKind::Semicolon
&& child.kind() != &NodeKind::Text('/'.into())
&& child.kind() != &SyntaxKind::Semicolon
&& child.kind() != &SyntaxKind::Text('/'.into())
&& (ahead.is_none() || change.replaced.start > child_span.end)
&& !ahead.map_or(false, Ahead::is_compulsory)
{
@ -153,8 +153,8 @@ fn try_reparse(
let superseded_span = pos.offset..pos.offset + prev_len;
let func: Option<ReparseMode> = match child.kind() {
NodeKind::CodeBlock => Some(ReparseMode::Code),
NodeKind::ContentBlock => Some(ReparseMode::Content),
SyntaxKind::CodeBlock => Some(ReparseMode::Code),
SyntaxKind::ContentBlock => Some(ReparseMode::Content),
_ => None,
};
@ -177,7 +177,7 @@ fn try_reparse(
// Make sure this is a markup node and that we may replace. If so, save
// the current indent.
let min_indent = match node.kind() {
NodeKind::Markup { min_indent } if safe_to_replace => *min_indent,
SyntaxKind::Markup { min_indent } if safe_to_replace => *min_indent,
_ => return None,
};
@ -375,26 +375,28 @@ enum ReparseMode {
/// Whether changes _inside_ this node are safely encapsulated, so that only
/// this node must be reparsed.
fn is_bounded(kind: &NodeKind) -> bool {
fn is_bounded(kind: &SyntaxKind) -> bool {
matches!(
kind,
NodeKind::CodeBlock
| NodeKind::ContentBlock
| NodeKind::Linebreak
| NodeKind::SmartQuote { .. }
| NodeKind::BlockComment
| NodeKind::Space { .. }
| NodeKind::Escape(_)
| NodeKind::Shorthand(_)
SyntaxKind::CodeBlock
| SyntaxKind::ContentBlock
| SyntaxKind::Linebreak
| SyntaxKind::SmartQuote { .. }
| SyntaxKind::BlockComment
| SyntaxKind::Space { .. }
| SyntaxKind::Escape(_)
| SyntaxKind::Shorthand(_)
)
}
/// Whether `at_start` would still be true after this node given the
/// previous value of the property.
fn next_at_start(kind: &NodeKind, prev: bool) -> bool {
fn next_at_start(kind: &SyntaxKind, prev: bool) -> bool {
match kind {
NodeKind::Space { newlines: (1..) } => true,
NodeKind::Space { .. } | NodeKind::LineComment | NodeKind::BlockComment => prev,
SyntaxKind::Space { newlines: (1..) } => true,
SyntaxKind::Space { .. } | SyntaxKind::LineComment | SyntaxKind::BlockComment => {
prev
}
_ => false,
}
}

23
src/syntax/kind.rs
View File

@ -6,10 +6,9 @@ use crate::util::EcoString;
/// All syntactical building blocks that can be part of a Typst document.
///
/// Can be emitted as a token by the tokenizer or as part of a syntax node by
/// the parser.
/// Can be created by the tokenizer or by the parser.
#[derive(Debug, Clone, PartialEq)]
pub enum NodeKind {
pub enum SyntaxKind {
/// A line comment: `// ...`.
LineComment,
/// A block comment: `/* ... */`.
@ -254,7 +253,7 @@ pub enum NodeKind {
Error(ErrorPos, EcoString),
}
/// Fields of a [`Raw`](NodeKind::Raw) node.
/// Fields of the raw syntax kind.
#[derive(Debug, Clone, PartialEq, Hash)]
pub struct RawFields {
/// An optional identifier specifying the language to syntax-highlight in.
@ -293,7 +292,7 @@ pub enum ErrorPos {
End,
}
impl NodeKind {
impl SyntaxKind {
/// Whether this is trivia.
pub fn is_trivia(&self) -> bool {
self.is_space() || matches!(self, Self::LineComment | Self::BlockComment)
@ -311,18 +310,18 @@ impl NodeKind {
/// Whether this is an error.
pub fn is_error(&self) -> bool {
matches!(self, NodeKind::Error(_, _))
matches!(self, SyntaxKind::Error(_, _))
}
/// Does this node need termination through a semicolon or linebreak?
pub fn is_stmt(&self) -> bool {
matches!(
self,
NodeKind::LetBinding
| NodeKind::SetRule
| NodeKind::ShowRule
| NodeKind::ModuleImport
| NodeKind::ModuleInclude
SyntaxKind::LetBinding
| SyntaxKind::SetRule
| SyntaxKind::ShowRule
| SyntaxKind::ModuleImport
| SyntaxKind::ModuleInclude
)
}
@ -445,7 +444,7 @@ impl NodeKind {
}
}
impl Hash for NodeKind {
impl Hash for SyntaxKind {
fn hash<H: Hasher>(&self, state: &mut H) {
std::mem::discriminant(self).hash(state);
match self {

49
src/syntax/node.rs
View File

@ -1,9 +1,9 @@
use std::fmt::{self, Debug, Formatter};
use std::fmt::{self, Debug, Display, Formatter};
use std::ops::Range;
use std::sync::Arc;
use super::ast::TypedNode;
use super::{NodeKind, NumberingResult, SourceId, Span, Unnumberable};
use super::ast::AstNode;
use super::{SourceId, Span, SyntaxKind};
use crate::diag::SourceError;
/// A node in the untyped syntax tree.
@ -21,22 +21,22 @@ enum Repr {
impl SyntaxNode {
/// Create a new leaf node.
pub fn leaf(kind: NodeKind, len: usize) -> Self {
pub fn leaf(kind: SyntaxKind, len: usize) -> Self {
Self(Repr::Leaf(NodeData::new(kind, len)))
}
/// Create a new inner node with children.
pub fn inner(kind: NodeKind, children: Vec<SyntaxNode>) -> Self {
pub fn inner(kind: SyntaxKind, children: Vec<SyntaxNode>) -> Self {
Self(Repr::Inner(Arc::new(InnerNode::with_children(kind, children))))
}
/// The type of the node.
pub fn kind(&self) -> &NodeKind {
pub fn kind(&self) -> &SyntaxKind {
&self.data().kind
}
/// Take the kind out of the node.
pub fn take(self) -> NodeKind {
pub fn take(self) -> SyntaxKind {
match self.0 {
Repr::Leaf(leaf) => leaf.kind,
Repr::Inner(inner) => inner.data.kind.clone(),
@ -72,18 +72,18 @@ impl SyntaxNode {
/// Convert the node to a typed AST node.
pub fn cast<T>(&self) -> Option<T>
where
T: TypedNode,
T: AstNode,
{
T::from_untyped(self)
}
/// Get the first child that can cast to the AST type `T`.
pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> {
pub fn cast_first_child<T: AstNode>(&self) -> Option<T> {
self.children().find_map(Self::cast)
}
/// Get the last child that can cast to the AST type `T`.
pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> {
pub fn cast_last_child<T: AstNode>(&self) -> Option<T> {
self.children().rev().find_map(Self::cast)
}
@ -102,7 +102,7 @@ impl SyntaxNode {
}
match self.kind() {
NodeKind::Error(pos, message) => {
SyntaxKind::Error(pos, message) => {
vec![SourceError::new(self.span(), message.clone()).with_pos(*pos)]
}
_ => self
@ -114,7 +114,7 @@ impl SyntaxNode {
}
/// Change the type of the node.
pub(super) fn convert(&mut self, kind: NodeKind) {
pub(super) fn convert(&mut self, kind: SyntaxKind) {
match &mut self.0 {
Repr::Inner(inner) => {
let node = Arc::make_mut(inner);
@ -226,7 +226,7 @@ impl Debug for SyntaxNode {
impl Default for SyntaxNode {
fn default() -> Self {
Self::leaf(NodeKind::None, 0)
Self::leaf(SyntaxKind::None, 0)
}
}
@ -247,7 +247,7 @@ struct InnerNode {
impl InnerNode {
/// Create a new inner node with the given kind and children.
fn with_children(kind: NodeKind, children: Vec<SyntaxNode>) -> Self {
fn with_children(kind: SyntaxKind, children: Vec<SyntaxNode>) -> Self {
let mut len = 0;
let mut descendants = 1;
let mut erroneous = kind.is_error();
@ -305,7 +305,7 @@ impl InnerNode {
}
}
// Number this node itself.
// Number the node itself.
let mut start = within.start;
if range.is_none() {
let end = start + stride;
@ -480,7 +480,7 @@ impl PartialEq for InnerNode {
struct NodeData {
/// What kind of node this is (each kind would have its own struct in a
/// strongly typed AST).
kind: NodeKind,
kind: SyntaxKind,
/// The byte length of the node in the source.
len: usize,
/// The node's span.
@ -489,7 +489,7 @@ struct NodeData {
impl NodeData {
/// Create new node metadata.
fn new(kind: NodeKind, len: usize) -> Self {
fn new(kind: SyntaxKind, len: usize) -> Self {
Self { len, kind, span: Span::detached() }
}
@ -525,3 +525,18 @@ impl PartialEq for NodeData {
self.kind == other.kind && self.len == other.len
}
}
/// Result of numbering a node within an interval.
pub(super) type NumberingResult = Result<(), Unnumberable>;
/// Indicates that a node cannot be numbered within a given interval.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub(super) struct Unnumberable;
impl Display for Unnumberable {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
f.pad("cannot number within this interval")
}
}
impl std::error::Error for Unnumberable {}

94
src/syntax/parser.rs
View File

@ -2,7 +2,7 @@ use std::fmt::{self, Display, Formatter};
use std::mem;
use std::ops::Range;
use super::{ErrorPos, NodeKind, SyntaxNode, TokenMode, Tokens};
use super::{ErrorPos, SyntaxKind, SyntaxNode, TokenMode, Tokens};
use crate::util::{format_eco, EcoString};
/// A convenient token-based parser.
@ -12,7 +12,7 @@ pub struct Parser<'s> {
/// Whether we are at the end of the file or of a group.
eof: bool,
/// The current token.
current: Option<NodeKind>,
current: Option<SyntaxKind>,
/// The end byte index of the last non-trivia token.
prev_end: usize,
/// The start byte index of the peeked token.
@ -82,7 +82,7 @@ impl<'s> Parser<'s> {
}
/// Perform a subparse that wraps its result in a node with the given kind.
pub fn perform<F, T>(&mut self, kind: NodeKind, f: F) -> T
pub fn perform<F, T>(&mut self, kind: SyntaxKind, f: F) -> T
where
F: FnOnce(&mut Self) -> T,
{
@ -112,9 +112,9 @@ impl<'s> Parser<'s> {
/// Consume the current token and also trailing trivia.
pub fn eat(&mut self) {
self.stray_terminator |= match self.current {
Some(NodeKind::RightParen) => !self.inside(Group::Paren),
Some(NodeKind::RightBracket) => !self.inside(Group::Bracket),
Some(NodeKind::RightBrace) => !self.inside(Group::Brace),
Some(SyntaxKind::RightParen) => !self.inside(Group::Paren),
Some(SyntaxKind::RightBracket) => !self.inside(Group::Bracket),
Some(SyntaxKind::RightBrace) => !self.inside(Group::Brace),
_ => false,
};
@ -132,7 +132,7 @@ impl<'s> Parser<'s> {
}
/// Consume the current token if it is the given one.
pub fn eat_if(&mut self, kind: NodeKind) -> bool {
pub fn eat_if(&mut self, kind: SyntaxKind) -> bool {
let at = self.at(kind);
if at {
self.eat();
@ -143,7 +143,7 @@ impl<'s> Parser<'s> {
/// Eat tokens while the condition is true.
pub fn eat_while<F>(&mut self, mut f: F)
where
F: FnMut(&NodeKind) -> bool,
F: FnMut(&SyntaxKind) -> bool,
{
while self.peek().map_or(false, |t| f(t)) {
self.eat();
@ -152,7 +152,7 @@ impl<'s> Parser<'s> {
/// Consume the current token if it is the given one and produce an error if
/// not.
pub fn expect(&mut self, kind: NodeKind) -> ParseResult {
pub fn expect(&mut self, kind: SyntaxKind) -> ParseResult {
let at = self.peek() == Some(&kind);
if at {
self.eat();
@ -165,18 +165,18 @@ impl<'s> Parser<'s> {
/// Consume the current token, debug-asserting that it is the given one.
#[track_caller]
pub fn assert(&mut self, kind: NodeKind) {
pub fn assert(&mut self, kind: SyntaxKind) {
debug_assert_eq!(self.peek(), Some(&kind));
self.eat();
}
/// Whether the current token is of the given type.
pub fn at(&self, kind: NodeKind) -> bool {
pub fn at(&self, kind: SyntaxKind) -> bool {
self.peek() == Some(&kind)
}
/// Peek at the current token without consuming it.
pub fn peek(&self) -> Option<&NodeKind> {
pub fn peek(&self) -> Option<&SyntaxKind> {
if self.eof {
None
} else {
@ -186,7 +186,7 @@ impl<'s> Parser<'s> {
/// Peek at the current token, but only if it follows immediately after the
/// last one without any trivia in between.
pub fn peek_direct(&self) -> Option<&NodeKind> {
pub fn peek_direct(&self) -> Option<&SyntaxKind> {
if self.prev_end() == self.current_start() {
self.peek()
} else {
@ -249,12 +249,12 @@ impl<'s> Parser<'s> {
});
match kind {
Group::Brace => self.assert(NodeKind::LeftBrace),
Group::Bracket => self.assert(NodeKind::LeftBracket),
Group::Paren => self.assert(NodeKind::LeftParen),
Group::Strong => self.assert(NodeKind::Star),
Group::Emph => self.assert(NodeKind::Underscore),
Group::Math => self.assert(NodeKind::Dollar),
Group::Brace => self.assert(SyntaxKind::LeftBrace),
Group::Bracket => self.assert(SyntaxKind::LeftBracket),
Group::Paren => self.assert(SyntaxKind::LeftParen),
Group::Strong => self.assert(SyntaxKind::Star),
Group::Emph => self.assert(SyntaxKind::Underscore),
Group::Math => self.assert(SyntaxKind::Dollar),
Group::Expr => self.repeek(),
Group::Imports => self.repeek(),
}
@ -273,13 +273,13 @@ impl<'s> Parser<'s> {
// Eat the end delimiter if there is one.
if let Some((end, required)) = match group.kind {
Group::Brace => Some((NodeKind::RightBrace, true)),
Group::Bracket => Some((NodeKind::RightBracket, true)),
Group::Paren => Some((NodeKind::RightParen, true)),
Group::Strong => Some((NodeKind::Star, true)),
Group::Emph => Some((NodeKind::Underscore, true)),
Group::Math => Some((NodeKind::Dollar, true)),
Group::Expr => Some((NodeKind::Semicolon, false)),
Group::Brace => Some((SyntaxKind::RightBrace, true)),
Group::Bracket => Some((SyntaxKind::RightBracket, true)),
Group::Paren => Some((SyntaxKind::RightParen, true)),
Group::Strong => Some((SyntaxKind::Star, true)),
Group::Emph => Some((SyntaxKind::Underscore, true)),
Group::Math => Some((SyntaxKind::Dollar, true)),
Group::Expr => Some((SyntaxKind::Semicolon, false)),
Group::Imports => None,
} {
if self.current.as_ref() == Some(&end) {
@ -339,26 +339,26 @@ impl<'s> Parser<'s> {
/// group.
fn repeek(&mut self) {
self.eof = match &self.current {
Some(NodeKind::RightBrace) => self.inside(Group::Brace),
Some(NodeKind::RightBracket) => self.inside(Group::Bracket),
Some(NodeKind::RightParen) => self.inside(Group::Paren),
Some(NodeKind::Star) => self.inside(Group::Strong),
Some(NodeKind::Underscore) => self.inside(Group::Emph),
Some(NodeKind::Dollar) => self.inside(Group::Math),
Some(NodeKind::Semicolon) => self.inside(Group::Expr),
Some(NodeKind::From) => self.inside(Group::Imports),
Some(NodeKind::Space { newlines }) => self.space_ends_group(*newlines),
Some(SyntaxKind::RightBrace) => self.inside(Group::Brace),
Some(SyntaxKind::RightBracket) => self.inside(Group::Bracket),
Some(SyntaxKind::RightParen) => self.inside(Group::Paren),
Some(SyntaxKind::Star) => self.inside(Group::Strong),
Some(SyntaxKind::Underscore) => self.inside(Group::Emph),
Some(SyntaxKind::Dollar) => self.inside(Group::Math),
Some(SyntaxKind::Semicolon) => self.inside(Group::Expr),
Some(SyntaxKind::From) => self.inside(Group::Imports),
Some(SyntaxKind::Space { newlines }) => self.space_ends_group(*newlines),
Some(_) => false,
None => true,
};
}
/// Returns whether the given type can be skipped over.
fn is_trivia(&self, token: &NodeKind) -> bool {
fn is_trivia(&self, token: &SyntaxKind) -> bool {
match token {
NodeKind::Space { newlines } => !self.space_ends_group(*newlines),
NodeKind::LineComment => true,
NodeKind::BlockComment => true,
SyntaxKind::Space { newlines } => !self.space_ends_group(*newlines),
SyntaxKind::LineComment => true,
SyntaxKind::BlockComment => true,
_ => false,
}
}
@ -378,7 +378,7 @@ impl<'s> Parser<'s> {
!= Some(Group::Brace)
|| !matches!(
self.tokens.clone().next(),
Some(NodeKind::Else | NodeKind::Dot)
Some(SyntaxKind::Else | SyntaxKind::Dot)
)
}
_ => false,
@ -401,7 +401,7 @@ impl Parser<'_> {
pub fn unexpected(&mut self) {
if let Some(found) = self.peek() {
let msg = format_eco!("unexpected {}", found.name());
let error = NodeKind::Error(ErrorPos::Full, msg);
let error = SyntaxKind::Error(ErrorPos::Full, msg);
self.perform(error, Self::eat);
}
}
@ -415,7 +415,7 @@ impl Parser<'_> {
/// Insert an error message that `what` was expected at the marker position.
pub fn expected_at(&mut self, marker: Marker, what: &str) {
let msg = format_eco!("expected {}", what);
let error = NodeKind::Error(ErrorPos::Full, msg);
let error = SyntaxKind::Error(ErrorPos::Full, msg);
self.children.insert(marker.0, SyntaxNode::leaf(error, 0));
}
@ -425,7 +425,7 @@ impl Parser<'_> {
match self.peek() {
Some(found) => {
let msg = format_eco!("expected {}, found {}", thing, found.name());
let error = NodeKind::Error(ErrorPos::Full, msg);
let error = SyntaxKind::Error(ErrorPos::Full, msg);
self.perform(error, Self::eat);
}
None => self.expected(thing),
@ -449,7 +449,7 @@ impl Marker {
}
/// Convert the child directly after marker.
pub fn convert(self, p: &mut Parser, kind: NodeKind) {
pub fn convert(self, p: &mut Parser, kind: SyntaxKind) {
if let Some(child) = p.children.get_mut(self.0) {
child.convert(kind);
}
@ -457,7 +457,7 @@ impl Marker {
/// Perform a subparse that wraps all children after the marker in a node
/// with the given kind.
pub fn perform<T, F>(self, p: &mut Parser, kind: NodeKind, f: F) -> T
pub fn perform<T, F>(self, p: &mut Parser, kind: SyntaxKind, f: F) -> T
where
F: FnOnce(&mut Parser) -> T,
{
@ -468,7 +468,7 @@ impl Marker {
/// Wrap all children after the marker (excluding trailing trivia) in a node
/// with the given `kind`.
pub fn end(self, p: &mut Parser, kind: NodeKind) {
pub fn end(self, p: &mut Parser, kind: SyntaxKind) {
let until = p.trivia_start().0.max(self.0);
let children = p.children.drain(self.0..until).collect();
p.children.insert(self.0, SyntaxNode::inner(kind, children));
@ -496,7 +496,7 @@ impl Marker {
msg.push_str(", found ");
msg.push_str(child.kind().name());
}
let error = NodeKind::Error(ErrorPos::Full, msg);
let error = SyntaxKind::Error(ErrorPos::Full, msg);
let inner = mem::take(child);
*child = SyntaxNode::inner(error, vec![inner]);
}

435
src/syntax/parsing.rs
View File

@ -2,7 +2,7 @@ use std::collections::HashSet;
use super::ast::{Assoc, BinOp, UnOp};
use super::{
ErrorPos, Group, Marker, NodeKind, ParseError, ParseResult, Parser, SyntaxNode,
ErrorPos, Group, Marker, ParseError, ParseResult, Parser, SyntaxKind, SyntaxNode,
TokenMode,
};
use crate::util::EcoString;
@ -17,7 +17,7 @@ pub fn parse(text: &str) -> SyntaxNode {
/// Parse code directly, only used for syntax highlighting.
pub fn parse_code(text: &str) -> SyntaxNode {
let mut p = Parser::new(text, TokenMode::Code);
p.perform(NodeKind::CodeBlock, code);
p.perform(SyntaxKind::CodeBlock, code);
p.finish().into_iter().next().unwrap()
}
@ -30,7 +30,7 @@ pub(crate) fn reparse_code_block(
end_pos: usize,
) -> Option<(Vec<SyntaxNode>, bool, usize)> {
let mut p = Parser::with_prefix(prefix, text, TokenMode::Code);
if !p.at(NodeKind::LeftBrace) {
if !p.at(SyntaxKind::LeftBrace) {
return None;
}
@ -54,7 +54,7 @@ pub(crate) fn reparse_content_block(
end_pos: usize,
) -> Option<(Vec<SyntaxNode>, bool, usize)> {
let mut p = Parser::with_prefix(prefix, text, TokenMode::Code);
if !p.at(NodeKind::LeftBracket) {
if !p.at(SyntaxKind::LeftBracket) {
return None;
}
@ -90,7 +90,7 @@ pub(crate) fn reparse_markup_elements(
let mut stopped = false;
'outer: while !p.eof() {
if let Some(NodeKind::Space { newlines: (1..) }) = p.peek() {
if let Some(SyntaxKind::Space { newlines: (1..) }) = p.peek() {
if p.column(p.current_end()) < min_indent {
return None;
}
@ -147,7 +147,7 @@ pub(crate) fn reparse_markup_elements(
/// If `at_start` is true, things like headings that may only appear at the
/// beginning of a line or content block are initially allowed.
fn markup(p: &mut Parser, mut at_start: bool) {
p.perform(NodeKind::Markup { min_indent: 0 }, |p| {
p.perform(SyntaxKind::Markup { min_indent: 0 }, |p| {
while !p.eof() {
markup_node(p, &mut at_start);
}
@ -157,8 +157,8 @@ fn markup(p: &mut Parser, mut at_start: bool) {
/// Parse markup that stays right of the given `column`.
fn markup_indented(p: &mut Parser, min_indent: usize) {
p.eat_while(|t| match t {
NodeKind::Space { newlines } => *newlines == 0,
NodeKind::LineComment | NodeKind::BlockComment => true,
SyntaxKind::Space { newlines } => *newlines == 0,
SyntaxKind::LineComment | SyntaxKind::BlockComment => true,
_ => false,
});
@ -167,7 +167,7 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
while !p.eof() {
match p.peek() {
Some(NodeKind::Space { newlines: (1..) })
Some(SyntaxKind::Space { newlines: (1..) })
if p.column(p.current_end()) < min_indent =>
{
break;
@ -178,24 +178,24 @@ fn markup_indented(p: &mut Parser, min_indent: usize) {
markup_node(p, &mut at_start);
}
marker.end(p, NodeKind::Markup { min_indent });
marker.end(p, SyntaxKind::Markup { min_indent });
}
/// Parse a line of markup that can prematurely end if `f` returns true.
fn markup_line<F>(p: &mut Parser, mut f: F)
where
F: FnMut(&NodeKind) -> bool,
F: FnMut(&SyntaxKind) -> bool,
{
p.eat_while(|t| match t {
NodeKind::Space { newlines } => *newlines == 0,
NodeKind::LineComment | NodeKind::BlockComment => true,
SyntaxKind::Space { newlines } => *newlines == 0,
SyntaxKind::LineComment | SyntaxKind::BlockComment => true,
_ => false,
});
p.perform(NodeKind::Markup { min_indent: usize::MAX }, |p| {
p.perform(SyntaxKind::Markup { min_indent: usize::MAX }, |p| {
let mut at_start = false;
while let Some(kind) = p.peek() {
if let NodeKind::Space { newlines: (1..) } = kind {
if let SyntaxKind::Space { newlines: (1..) } = kind {
break;
}
@ -212,68 +212,68 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
let Some(token) = p.peek() else { return };
match token {
// Whitespace.
NodeKind::Space { newlines } => {
SyntaxKind::Space { newlines } => {
*at_start |= *newlines > 0;
p.eat();
return;
}
// Comments.
NodeKind::LineComment | NodeKind::BlockComment => {
SyntaxKind::LineComment | SyntaxKind::BlockComment => {
p.eat();
return;
}
// Text and markup.
NodeKind::Text(_)
| NodeKind::Linebreak
| NodeKind::SmartQuote { .. }
| NodeKind::Escape(_)
| NodeKind::Shorthand(_)
| NodeKind::Link(_)
| NodeKind::Raw(_)
| NodeKind::Label(_)
| NodeKind::Ref(_) => p.eat(),
SyntaxKind::Text(_)
| SyntaxKind::Linebreak
| SyntaxKind::SmartQuote { .. }
| SyntaxKind::Escape(_)
| SyntaxKind::Shorthand(_)
| SyntaxKind::Link(_)
| SyntaxKind::Raw(_)
| SyntaxKind::Label(_)
| SyntaxKind::Ref(_) => p.eat(),
// Math.
NodeKind::Dollar => math(p),
SyntaxKind::Dollar => math(p),
// Strong, emph, heading.
NodeKind::Star => strong(p),
NodeKind::Underscore => emph(p),
NodeKind::Eq => heading(p, *at_start),
SyntaxKind::Star => strong(p),
SyntaxKind::Underscore => emph(p),
SyntaxKind::Eq => heading(p, *at_start),
// Lists.
NodeKind::Minus => list_item(p, *at_start),
NodeKind::Plus | NodeKind::EnumNumbering(_) => enum_item(p, *at_start),
NodeKind::Slash => {
SyntaxKind::Minus => list_item(p, *at_start),
SyntaxKind::Plus | SyntaxKind::EnumNumbering(_) => enum_item(p, *at_start),
SyntaxKind::Slash => {
desc_item(p, *at_start).ok();
}
NodeKind::Colon => {
SyntaxKind::Colon => {
let marker = p.marker();
p.eat();
marker.convert(p, NodeKind::Text(':'.into()));
marker.convert(p, SyntaxKind::Text(':'.into()));
}
// Hashtag + keyword / identifier.
NodeKind::Ident(_)
| NodeKind::Let
| NodeKind::Set
| NodeKind::Show
| NodeKind::If
| NodeKind::While
| NodeKind::For
| NodeKind::Import
| NodeKind::Include
| NodeKind::Break
| NodeKind::Continue
| NodeKind::Return => markup_expr(p),
SyntaxKind::Ident(_)
| SyntaxKind::Let
| SyntaxKind::Set
| SyntaxKind::Show
| SyntaxKind::If
| SyntaxKind::While
| SyntaxKind::For
| SyntaxKind::Import
| SyntaxKind::Include
| SyntaxKind::Break
| SyntaxKind::Continue
| SyntaxKind::Return => markup_expr(p),
// Code and content block.
NodeKind::LeftBrace => code_block(p),
NodeKind::LeftBracket => content_block(p),
SyntaxKind::LeftBrace => code_block(p),
SyntaxKind::LeftBracket => content_block(p),
NodeKind::Error(_, _) => p.eat(),
SyntaxKind::Error(_, _) => p.eat(),
_ => p.unexpected(),
};
@ -281,7 +281,7 @@ fn markup_node(p: &mut Parser, at_start: &mut bool) {
}
fn strong(p: &mut Parser) {
p.perform(NodeKind::Strong, |p| {
p.perform(SyntaxKind::Strong, |p| {
p.start_group(Group::Strong);
markup(p, false);
p.end_group();
@ -289,7 +289,7 @@ fn strong(p: &mut Parser) {
}
fn emph(p: &mut Parser) {
p.perform(NodeKind::Emph, |p| {
p.perform(SyntaxKind::Emph, |p| {
p.start_group(Group::Emph);
markup(p, false);
p.end_group();
@ -299,30 +299,30 @@ fn emph(p: &mut Parser) {
fn heading(p: &mut Parser, at_start: bool) {
let marker = p.marker();
let current_start = p.current_start();
p.assert(NodeKind::Eq);
while p.eat_if(NodeKind::Eq) {}
p.assert(SyntaxKind::Eq);
while p.eat_if(SyntaxKind::Eq) {}
if at_start && p.peek().map_or(true, |kind| kind.is_space()) {
p.eat_while(|kind| *kind == NodeKind::Space { newlines: 0 });
markup_line(p, |kind| matches!(kind, NodeKind::Label(_)));
marker.end(p, NodeKind::Heading);
p.eat_while(|kind| *kind == SyntaxKind::Space { newlines: 0 });
markup_line(p, |kind| matches!(kind, SyntaxKind::Label(_)));
marker.end(p, SyntaxKind::Heading);
} else {
let text = p.get(current_start..p.prev_end()).into();
marker.convert(p, NodeKind::Text(text));
marker.convert(p, SyntaxKind::Text(text));
}
}
fn list_item(p: &mut Parser, at_start: bool) {
let marker = p.marker();
let text: EcoString = p.peek_src().into();
p.assert(NodeKind::Minus);
p.assert(SyntaxKind::Minus);
let min_indent = p.column(p.prev_end());
if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() {
if at_start && p.eat_if(SyntaxKind::Space { newlines: 0 }) && !p.eof() {
markup_indented(p, min_indent);
marker.end(p, NodeKind::ListItem);
marker.end(p, SyntaxKind::ListItem);
} else {
marker.convert(p, NodeKind::Text(text));
marker.convert(p, SyntaxKind::Text(text));
}
}
@ -332,11 +332,11 @@ fn enum_item(p: &mut Parser, at_start: bool) {
p.eat();
let min_indent = p.column(p.prev_end());
if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() {
if at_start && p.eat_if(SyntaxKind::Space { newlines: 0 }) && !p.eof() {
markup_indented(p, min_indent);
marker.end(p, NodeKind::EnumItem);
marker.end(p, SyntaxKind::EnumItem);
} else {
marker.convert(p, NodeKind::Text(text));
marker.convert(p, SyntaxKind::Text(text));
}
}
@ -346,13 +346,13 @@ fn desc_item(p: &mut Parser, at_start: bool) -> ParseResult {
p.eat();
let min_indent = p.column(p.prev_end());
if at_start && p.eat_if(NodeKind::Space { newlines: 0 }) && !p.eof() {
markup_line(p, |node| matches!(node, NodeKind::Colon));
p.expect(NodeKind::Colon)?;
if at_start && p.eat_if(SyntaxKind::Space { newlines: 0 }) && !p.eof() {
markup_line(p, |node| matches!(node, SyntaxKind::Colon));
p.expect(SyntaxKind::Colon)?;
markup_indented(p, min_indent);
marker.end(p, NodeKind::DescItem);
marker.end(p, SyntaxKind::DescItem);
} else {
marker.convert(p, NodeKind::Text(text));
marker.convert(p, SyntaxKind::Text(text));
}
Ok(())
@ -363,11 +363,11 @@ fn markup_expr(p: &mut Parser) {
let stmt = matches!(
p.peek(),
Some(
NodeKind::Let
| NodeKind::Set
| NodeKind::Show
| NodeKind::Import
| NodeKind::Include
SyntaxKind::Let
| SyntaxKind::Set
| SyntaxKind::Show
| SyntaxKind::Import
| SyntaxKind::Include
)
);
@ -380,7 +380,7 @@ fn markup_expr(p: &mut Parser) {
}
fn math(p: &mut Parser) {
p.perform(NodeKind::Math, |p| {
p.perform(SyntaxKind::Math, |p| {
p.start_group(Group::Math);
while !p.eof() {
math_node(p);
@ -393,20 +393,20 @@ fn math_node(p: &mut Parser) {
math_node_prec(p, 0, None)
}
fn math_node_prec(p: &mut Parser, min_prec: usize, stop: Option<NodeKind>) {
fn math_node_prec(p: &mut Parser, min_prec: usize, stop: Option<SyntaxKind>) {
let marker = p.marker();
math_primary(p);
loop {
let (kind, mut prec, assoc, stop) = match p.peek() {
v if v == stop.as_ref() => break,
Some(NodeKind::Underscore) => {
(NodeKind::Script, 2, Assoc::Right, Some(NodeKind::Hat))
Some(SyntaxKind::Underscore) => {
(SyntaxKind::Script, 2, Assoc::Right, Some(SyntaxKind::Hat))
}
Some(NodeKind::Hat) => {
(NodeKind::Script, 2, Assoc::Right, Some(NodeKind::Underscore))
Some(SyntaxKind::Hat) => {
(SyntaxKind::Script, 2, Assoc::Right, Some(SyntaxKind::Underscore))
}
Some(NodeKind::Slash) => (NodeKind::Frac, 1, Assoc::Left, None),
Some(SyntaxKind::Slash) => (SyntaxKind::Frac, 1, Assoc::Left, None),
_ => break,
};
@ -424,7 +424,7 @@ fn math_node_prec(p: &mut Parser, min_prec: usize, stop: Option<NodeKind>) {
// Allow up to two different scripts. We do not risk encountering the
// previous script kind again here due to right-associativity.
if p.eat_if(NodeKind::Underscore) || p.eat_if(NodeKind::Hat) {
if p.eat_if(SyntaxKind::Underscore) || p.eat_if(SyntaxKind::Hat) {
math_node_prec(p, prec, None);
}
@ -437,42 +437,42 @@ fn math_primary(p: &mut Parser) {
let Some(token) = p.peek() else { return };
match token {
// Spaces, atoms and expressions.
NodeKind::Space { .. }
| NodeKind::Linebreak
| NodeKind::Escape(_)
| NodeKind::Atom(_)
| NodeKind::Ident(_) => p.eat(),
SyntaxKind::Space { .. }
| SyntaxKind::Linebreak
| SyntaxKind::Escape(_)
| SyntaxKind::Atom(_)
| SyntaxKind::Ident(_) => p.eat(),
// Groups.
NodeKind::LeftParen => math_group(p, Group::Paren, '(', ')'),
NodeKind::LeftBracket => math_group(p, Group::Bracket, '[', ']'),
NodeKind::LeftBrace => math_group(p, Group::Brace, '{', '}'),
SyntaxKind::LeftParen => math_group(p, Group::Paren, '(', ')'),
SyntaxKind::LeftBracket => math_group(p, Group::Bracket, '[', ']'),
SyntaxKind::LeftBrace => math_group(p, Group::Brace, '{', '}'),
// Alignment indactor.
NodeKind::Amp => math_align(p),
SyntaxKind::Amp => math_align(p),
_ => p.unexpected(),
}
}
fn math_group(p: &mut Parser, group: Group, l: char, r: char) {
p.perform(NodeKind::Math, |p| {
p.perform(SyntaxKind::Math, |p| {
let marker = p.marker();
p.start_group(group);
marker.convert(p, NodeKind::Atom(l.into()));
marker.convert(p, SyntaxKind::Atom(l.into()));
while !p.eof() {
math_node(p);
}
let marker = p.marker();
p.end_group();
marker.convert(p, NodeKind::Atom(r.into()));
marker.convert(p, SyntaxKind::Atom(r.into()));
})
}
fn math_align(p: &mut Parser) {
p.perform(NodeKind::Align, |p| {
p.assert(NodeKind::Amp);
while p.eat_if(NodeKind::Amp) {}
p.perform(SyntaxKind::Align, |p| {
p.assert(SyntaxKind::Amp);
while p.eat_if(SyntaxKind::Amp) {}
})
}
@ -496,15 +496,15 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
p.eat();
let prec = op.precedence();
expr_prec(p, atomic, prec)?;
marker.end(p, NodeKind::Unary);
marker.end(p, SyntaxKind::Unary);
}
_ => primary(p, atomic)?,
};
loop {
// Parenthesis or bracket means this is a function call.
if let Some(NodeKind::LeftParen | NodeKind::LeftBracket) = p.peek_direct() {
marker.perform(p, NodeKind::FuncCall, args)?;
if let Some(SyntaxKind::LeftParen | SyntaxKind::LeftBracket) = p.peek_direct() {
marker.perform(p, SyntaxKind::FuncCall, args)?;
continue;
}
@ -513,18 +513,19 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
}
// Method call or field access.
if p.eat_if(NodeKind::Dot) {
if p.eat_if(SyntaxKind::Dot) {
ident(p)?;
if let Some(NodeKind::LeftParen | NodeKind::LeftBracket) = p.peek_direct() {
marker.perform(p, NodeKind::MethodCall, args)?;
if let Some(SyntaxKind::LeftParen | SyntaxKind::LeftBracket) = p.peek_direct()
{
marker.perform(p, SyntaxKind::MethodCall, args)?;
} else {
marker.end(p, NodeKind::FieldAccess);
marker.end(p, SyntaxKind::FieldAccess);
}
continue;
}
let op = if p.eat_if(NodeKind::Not) {
if p.at(NodeKind::In) {
let op = if p.eat_if(SyntaxKind::Not) {
if p.at(SyntaxKind::In) {
BinOp::NotIn
} else {
p.expected("keyword `in`");
@ -549,7 +550,7 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
Assoc::Right => {}
}
marker.perform(p, NodeKind::Binary, |p| expr_prec(p, atomic, prec))?;
marker.perform(p, SyntaxKind::Binary, |p| expr_prec(p, atomic, prec))?;
}
Ok(())
@ -562,39 +563,39 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
match p.peek() {
// Things that start with an identifier.
Some(NodeKind::Ident(_)) => {
Some(SyntaxKind::Ident(_)) => {
let marker = p.marker();
p.eat();
// Arrow means this is a closure's lone parameter.
if !atomic && p.at(NodeKind::Arrow) {
marker.end(p, NodeKind::Params);
p.assert(NodeKind::Arrow);
marker.perform(p, NodeKind::Closure, expr)
if !atomic && p.at(SyntaxKind::Arrow) {
marker.end(p, SyntaxKind::Params);
p.assert(SyntaxKind::Arrow);
marker.perform(p, SyntaxKind::Closure, expr)
} else {
Ok(())
}
}
// Structures.
Some(NodeKind::LeftParen) => parenthesized(p, atomic),
Some(NodeKind::LeftBrace) => Ok(code_block(p)),
Some(NodeKind::LeftBracket) => Ok(content_block(p)),
Some(SyntaxKind::LeftParen) => parenthesized(p, atomic),
Some(SyntaxKind::LeftBrace) => Ok(code_block(p)),
Some(SyntaxKind::LeftBracket) => Ok(content_block(p)),
// Keywords.
Some(NodeKind::Let) => let_binding(p),
Some(NodeKind::Set) => set_rule(p),
Some(NodeKind::Show) => show_rule(p),
Some(NodeKind::If) => conditional(p),
Some(NodeKind::While) => while_loop(p),
Some(NodeKind::For) => for_loop(p),
Some(NodeKind::Import) => module_import(p),
Some(NodeKind::Include) => module_include(p),
Some(NodeKind::Break) => break_stmt(p),
Some(NodeKind::Continue) => continue_stmt(p),
Some(NodeKind::Return) => return_stmt(p),
Some(SyntaxKind::Let) => let_binding(p),
Some(SyntaxKind::Set) => set_rule(p),
Some(SyntaxKind::Show) => show_rule(p),
Some(SyntaxKind::If) => conditional(p),
Some(SyntaxKind::While) => while_loop(p),
Some(SyntaxKind::For) => for_loop(p),
Some(SyntaxKind::Import) => module_import(p),
Some(SyntaxKind::Include) => module_include(p),
Some(SyntaxKind::Break) => break_stmt(p),
Some(SyntaxKind::Continue) => continue_stmt(p),
Some(SyntaxKind::Return) => return_stmt(p),
Some(NodeKind::Error(_, _)) => {
Some(SyntaxKind::Error(_, _)) => {
p.eat();
Err(ParseError)
}
@ -610,13 +611,13 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
fn literal(p: &mut Parser) -> bool {
match p.peek() {
Some(
NodeKind::None
| NodeKind::Auto
| NodeKind::Int(_)
| NodeKind::Float(_)
| NodeKind::Bool(_)
| NodeKind::Numeric(_, _)
| NodeKind::Str(_),
SyntaxKind::None
| SyntaxKind::Auto
| SyntaxKind::Int(_)
| SyntaxKind::Float(_)
| SyntaxKind::Bool(_)
| SyntaxKind::Numeric(_, _)
| SyntaxKind::Str(_),
) => {
p.eat();
true
@ -627,7 +628,7 @@ fn literal(p: &mut Parser) -> bool {
fn ident(p: &mut Parser) -> ParseResult {
match p.peek() {
Some(NodeKind::Ident(_)) => {
Some(SyntaxKind::Ident(_)) => {
p.eat();
Ok(())
}
@ -647,7 +648,7 @@ fn parenthesized(p: &mut Parser, atomic: bool) -> ParseResult {
let marker = p.marker();
p.start_group(Group::Paren);
let colon = p.eat_if(NodeKind::Colon);
let colon = p.eat_if(SyntaxKind::Colon);
let kind = collection(p, true).0;
p.end_group();
@ -658,15 +659,15 @@ fn parenthesized(p: &mut Parser, atomic: bool) -> ParseResult {
}
// Arrow means this is a closure's parameter list.
if !atomic && p.at(NodeKind::Arrow) {
if !atomic && p.at(SyntaxKind::Arrow) {
params(p, marker);
p.assert(NodeKind::Arrow);
return marker.perform(p, NodeKind::Closure, expr);
p.assert(SyntaxKind::Arrow);
return marker.perform(p, SyntaxKind::Closure, expr);
}
// Transform into the identified collection.
match kind {
CollectionKind::Group => marker.end(p, NodeKind::Parenthesized),
CollectionKind::Group => marker.end(p, SyntaxKind::Parenthesized),
CollectionKind::Positional => array(p, marker),
CollectionKind::Named => dict(p, marker),
}
@ -698,14 +699,14 @@ fn collection(p: &mut Parser, keyed: bool) -> (CollectionKind, usize) {
while !p.eof() {
let Ok(item_kind) = item(p, keyed) else {
p.eat_if(NodeKind::Comma);
p.eat_if(SyntaxKind::Comma);
collection_kind = Some(CollectionKind::Group);
continue;
};
match item_kind {
NodeKind::Spread => can_group = false,
NodeKind::Named if collection_kind.is_none() => {
SyntaxKind::Spread => can_group = false,
SyntaxKind::Named if collection_kind.is_none() => {
collection_kind = Some(CollectionKind::Named);
can_group = false;
}
@ -725,7 +726,7 @@ fn collection(p: &mut Parser, keyed: bool) -> (CollectionKind, usize) {
break;
}
if p.eat_if(NodeKind::Comma) {
if p.eat_if(SyntaxKind::Comma) {
can_group = false;
} else {
missing_coma = Some(p.trivia_start());
@ -741,24 +742,24 @@ fn collection(p: &mut Parser, keyed: bool) -> (CollectionKind, usize) {
(kind, items)
}
fn item(p: &mut Parser, keyed: bool) -> ParseResult<NodeKind> {
fn item(p: &mut Parser, keyed: bool) -> ParseResult<SyntaxKind> {
let marker = p.marker();
if p.eat_if(NodeKind::Dots) {
marker.perform(p, NodeKind::Spread, expr)?;
return Ok(NodeKind::Spread);
if p.eat_if(SyntaxKind::Dots) {
marker.perform(p, SyntaxKind::Spread, expr)?;
return Ok(SyntaxKind::Spread);
}
expr(p)?;
if p.at(NodeKind::Colon) {
if p.at(SyntaxKind::Colon) {
match marker.after(p).map(|c| c.kind()) {
Some(NodeKind::Ident(_)) => {
Some(SyntaxKind::Ident(_)) => {
p.eat();
marker.perform(p, NodeKind::Named, expr)?;
marker.perform(p, SyntaxKind::Named, expr)?;
}
Some(NodeKind::Str(_)) if keyed => {
Some(SyntaxKind::Str(_)) if keyed => {
p.eat();
marker.perform(p, NodeKind::Keyed, expr)?;
marker.perform(p, SyntaxKind::Keyed, expr)?;
}
kind => {
let mut msg = EcoString::from("expected identifier");
@ -769,34 +770,34 @@ fn item(p: &mut Parser, keyed: bool) -> ParseResult<NodeKind> {
msg.push_str(", found ");
msg.push_str(kind.name());
}
let error = NodeKind::Error(ErrorPos::Full, msg);
let error = SyntaxKind::Error(ErrorPos::Full, msg);
marker.end(p, error);
p.eat();
marker.perform(p, NodeKind::Named, expr).ok();
marker.perform(p, SyntaxKind::Named, expr).ok();
return Err(ParseError);
}
}
Ok(NodeKind::Named)
Ok(SyntaxKind::Named)
} else {
Ok(NodeKind::None)
Ok(SyntaxKind::None)
}
}
fn array(p: &mut Parser, marker: Marker) {
marker.filter_children(p, |x| match x.kind() {
NodeKind::Named | NodeKind::Keyed => Err("expected expression"),
SyntaxKind::Named | SyntaxKind::Keyed => Err("expected expression"),
_ => Ok(()),
});
marker.end(p, NodeKind::Array);
marker.end(p, SyntaxKind::Array);
}
fn dict(p: &mut Parser, marker: Marker) {
let mut used = HashSet::new();
marker.filter_children(p, |x| match x.kind() {
kind if kind.is_paren() => Ok(()),
NodeKind::Named | NodeKind::Keyed => {
if let Some(NodeKind::Ident(key) | NodeKind::Str(key)) =
SyntaxKind::Named | SyntaxKind::Keyed => {
if let Some(SyntaxKind::Ident(key) | SyntaxKind::Str(key)) =
x.children().next().map(|child| child.kind())
{
if !used.insert(key.clone()) {
@ -805,32 +806,32 @@ fn dict(p: &mut Parser, marker: Marker) {
}
Ok(())
}
NodeKind::Spread | NodeKind::Comma | NodeKind::Colon => Ok(()),
SyntaxKind::Spread | SyntaxKind::Comma | SyntaxKind::Colon => Ok(()),
_ => Err("expected named or keyed pair"),
});
marker.end(p, NodeKind::Dict);
marker.end(p, SyntaxKind::Dict);
}
fn params(p: &mut Parser, marker: Marker) {
marker.filter_children(p, |x| match x.kind() {
kind if kind.is_paren() => Ok(()),
NodeKind::Named | NodeKind::Ident(_) | NodeKind::Comma => Ok(()),
NodeKind::Spread
SyntaxKind::Named | SyntaxKind::Ident(_) | SyntaxKind::Comma => Ok(()),
SyntaxKind::Spread
if matches!(
x.children().last().map(|child| child.kind()),
Some(&NodeKind::Ident(_))
Some(&SyntaxKind::Ident(_))
) =>
{
Ok(())
}
_ => Err("expected identifier, named pair or argument sink"),
});
marker.end(p, NodeKind::Params);
marker.end(p, SyntaxKind::Params);
}
/// Parse a code block: `{...}`.
fn code_block(p: &mut Parser) {
p.perform(NodeKind::CodeBlock, |p| {
p.perform(SyntaxKind::CodeBlock, |p| {
p.start_group(Group::Brace);
code(p);
p.end_group();
@ -846,12 +847,12 @@ fn code(p: &mut Parser) {
p.end_group();
// Forcefully skip over newlines since the group's contents can't.
p.eat_while(NodeKind::is_space);
p.eat_while(SyntaxKind::is_space);
}
}
fn content_block(p: &mut Parser) {
p.perform(NodeKind::ContentBlock, |p| {
p.perform(SyntaxKind::ContentBlock, |p| {
p.start_group(Group::Bracket);
markup(p, true);
p.end_group();
@ -860,16 +861,16 @@ fn content_block(p: &mut Parser) {
fn args(p: &mut Parser) -> ParseResult {
match p.peek_direct() {
Some(NodeKind::LeftParen) => {}
Some(NodeKind::LeftBracket) => {}
Some(SyntaxKind::LeftParen) => {}
Some(SyntaxKind::LeftBracket) => {}
_ => {
p.expected_found("argument list");
return Err(ParseError);
}
}
p.perform(NodeKind::Args, |p| {
if p.at(NodeKind::LeftParen) {
p.perform(SyntaxKind::Args, |p| {
if p.at(SyntaxKind::LeftParen) {
let marker = p.marker();
p.start_group(Group::Paren);
collection(p, false);
@ -877,8 +878,8 @@ fn args(p: &mut Parser) -> ParseResult {
let mut used = HashSet::new();
marker.filter_children(p, |x| match x.kind() {
NodeKind::Named => {
if let Some(NodeKind::Ident(ident)) =
SyntaxKind::Named => {
if let Some(SyntaxKind::Ident(ident)) =
x.children().next().map(|child| child.kind())
{
if !used.insert(ident.clone()) {
@ -891,7 +892,7 @@ fn args(p: &mut Parser) -> ParseResult {
});
}
while p.peek_direct() == Some(&NodeKind::LeftBracket) {
while p.peek_direct() == Some(&SyntaxKind::LeftBracket) {
content_block(p);
}
});
@ -900,14 +901,14 @@ fn args(p: &mut Parser) -> ParseResult {
}
fn let_binding(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::LetBinding, |p| {
p.assert(NodeKind::Let);
p.perform(SyntaxKind::LetBinding, |p| {
p.assert(SyntaxKind::Let);
let marker = p.marker();
ident(p)?;
// If a parenthesis follows, this is a function definition.
let has_params = p.peek_direct() == Some(&NodeKind::LeftParen);
let has_params = p.peek_direct() == Some(&SyntaxKind::LeftParen);
if has_params {
let marker = p.marker();
p.start_group(Group::Paren);
@ -916,7 +917,7 @@ fn let_binding(p: &mut Parser) -> ParseResult {
params(p, marker);
}
if p.eat_if(NodeKind::Eq) {
if p.eat_if(SyntaxKind::Eq) {
expr(p)?;
} else if has_params {
// Function definitions must have a body.
@ -925,7 +926,7 @@ fn let_binding(p: &mut Parser) -> ParseResult {
// Rewrite into a closure expression if it's a function definition.
if has_params {
marker.end(p, NodeKind::Closure);
marker.end(p, SyntaxKind::Closure);
}
Ok(())
@ -933,11 +934,11 @@ fn let_binding(p: &mut Parser) -> ParseResult {
}
fn set_rule(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::SetRule, |p| {
p.assert(NodeKind::Set);
p.perform(SyntaxKind::SetRule, |p| {
p.assert(SyntaxKind::Set);
ident(p)?;
args(p)?;
if p.eat_if(NodeKind::If) {
if p.eat_if(SyntaxKind::If) {
expr(p)?;
}
Ok(())
@ -945,10 +946,10 @@ fn set_rule(p: &mut Parser) -> ParseResult {
}
fn show_rule(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::ShowRule, |p| {
p.assert(NodeKind::Show);
p.perform(SyntaxKind::ShowRule, |p| {
p.assert(SyntaxKind::Show);
expr(p)?;
if p.eat_if(NodeKind::Colon) {
if p.eat_if(SyntaxKind::Colon) {
expr(p)?;
}
Ok(())
@ -956,14 +957,14 @@ fn show_rule(p: &mut Parser) -> ParseResult {
}
fn conditional(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::Conditional, |p| {
p.assert(NodeKind::If);
p.perform(SyntaxKind::Conditional, |p| {
p.assert(SyntaxKind::If);
expr(p)?;
body(p)?;
if p.eat_if(NodeKind::Else) {
if p.at(NodeKind::If) {
if p.eat_if(SyntaxKind::Else) {
if p.at(SyntaxKind::If) {
conditional(p)?;
} else {
body(p)?;
@ -975,27 +976,27 @@ fn conditional(p: &mut Parser) -> ParseResult {
}
fn while_loop(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::WhileLoop, |p| {
p.assert(NodeKind::While);
p.perform(SyntaxKind::WhileLoop, |p| {
p.assert(SyntaxKind::While);
expr(p)?;
body(p)
})
}
fn for_loop(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::ForLoop, |p| {
p.assert(NodeKind::For);
p.perform(SyntaxKind::ForLoop, |p| {
p.assert(SyntaxKind::For);
for_pattern(p)?;
p.expect(NodeKind::In)?;
p.expect(SyntaxKind::In)?;
expr(p)?;
body(p)
})
}
fn for_pattern(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::ForPattern, |p| {
p.perform(SyntaxKind::ForPattern, |p| {
ident(p)?;
if p.eat_if(NodeKind::Comma) {
if p.eat_if(SyntaxKind::Comma) {
ident(p)?;
}
Ok(())
@ -1003,12 +1004,12 @@ fn for_pattern(p: &mut Parser) -> ParseResult {
}
fn module_import(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::ModuleImport, |p| {
p.assert(NodeKind::Import);
p.perform(SyntaxKind::ModuleImport, |p| {
p.assert(SyntaxKind::Import);
if !p.eat_if(NodeKind::Star) {
if !p.eat_if(SyntaxKind::Star) {
// This is the list of identifiers scenario.
p.perform(NodeKind::ImportItems, |p| {
p.perform(SyntaxKind::ImportItems, |p| {
p.start_group(Group::Imports);
let marker = p.marker();
let items = collection(p, false).1;
@ -1018,42 +1019,42 @@ fn module_import(p: &mut Parser) -> ParseResult {
p.end_group();
marker.filter_children(p, |n| match n.kind() {
NodeKind::Ident(_) | NodeKind::Comma => Ok(()),
SyntaxKind::Ident(_) | SyntaxKind::Comma => Ok(()),
_ => Err("expected identifier"),
});
});
};
p.expect(NodeKind::From)?;
p.expect(SyntaxKind::From)?;
expr(p)
})
}
fn module_include(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::ModuleInclude, |p| {
p.assert(NodeKind::Include);
p.perform(SyntaxKind::ModuleInclude, |p| {
p.assert(SyntaxKind::Include);
expr(p)
})
}
fn break_stmt(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::LoopBreak, |p| {
p.assert(NodeKind::Break);
p.perform(SyntaxKind::LoopBreak, |p| {
p.assert(SyntaxKind::Break);
Ok(())
})
}
fn continue_stmt(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::LoopContinue, |p| {
p.assert(NodeKind::Continue);
p.perform(SyntaxKind::LoopContinue, |p| {
p.assert(SyntaxKind::Continue);
Ok(())
})
}
fn return_stmt(p: &mut Parser) -> ParseResult {
p.perform(NodeKind::FuncReturn, |p| {
p.assert(NodeKind::Return);
if !p.at(NodeKind::Comma) && !p.eof() {
p.perform(SyntaxKind::FuncReturn, |p| {
p.assert(SyntaxKind::Return);
if !p.at(SyntaxKind::Comma) && !p.eof() {
expr(p)?;
}
Ok(())
@ -1062,8 +1063,8 @@ fn return_stmt(p: &mut Parser) -> ParseResult {
fn body(p: &mut Parser) -> ParseResult {
match p.peek() {
Some(NodeKind::LeftBracket) => Ok(content_block(p)),
Some(NodeKind::LeftBrace) => Ok(code_block(p)),
Some(SyntaxKind::LeftBracket) => Ok(content_block(p)),
Some(SyntaxKind::LeftBrace) => Ok(code_block(p)),
_ => {
p.expected("body");
Err(ParseError)

157
src/syntax/span.rs
View File

@ -1,9 +1,79 @@
use std::fmt::{self, Debug, Display, Formatter};
use std::fmt::{self, Debug, Formatter};
use std::num::NonZeroU64;
use std::ops::Range;
use super::SourceId;
/// A unique identifier for a syntax node.
///
/// This is used throughout the compiler to track which source section an error
/// or element stems from. Can be [mapped back](super::Source::range) to a byte
/// range for user facing display.
///
/// During editing, the span values stay mostly stable, even for nodes behind an
/// insertion. This is not true for simple ranges as they would shift. Spans can
/// be used as inputs to memoized functions without hurting cache performance
/// when text is inserted somewhere in the document other than the end.
///
/// Span ids are ordered in the syntax tree to enable quickly finding the node
/// with some id:
/// - The id of a parent is always smaller than the ids of any of its children.
/// - The id of a node is always greater than any id in the subtrees of any left
/// sibling and smaller than any id in the subtrees of any right sibling.
///
/// This type takes up 8 bytes and is null-optimized (i.e. `Option<Span>` also
/// takes 8 bytes).
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub struct Span(NonZeroU64);
impl Span {
// Data layout:
// | 16 bits source id | 48 bits number |
// Number of bits for and minimum and maximum numbers assignable to spans.
const BITS: usize = 48;
const DETACHED: u64 = 1;
/// The full range of numbers available for span numbering.
pub const FULL: Range<u64> = 2..(1 << Self::BITS);
/// Create a new span from a source id and a unique number.
///
/// Panics if the `number` is not contained in `FULL`.
pub const fn new(id: SourceId, number: u64) -> Self {
assert!(
Self::FULL.start <= number && number < Self::FULL.end,
"span number outside valid range"
);
let bits = ((id.into_u16() as u64) << Self::BITS) | number;
Self(to_non_zero(bits))
}
/// A span that does not point into any source file.
pub const fn detached() -> Self {
Self(to_non_zero(Self::DETACHED))
}
/// The id of the source file the span points into.
pub const fn source(self) -> SourceId {
SourceId::from_u16((self.0.get() >> Self::BITS) as u16)
}
/// The unique number of the span within its source file.
pub const fn number(self) -> u64 {
self.0.get() & ((1 << Self::BITS) - 1)
}
}
/// Convert to a non zero u64.
const fn to_non_zero(v: u64) -> NonZeroU64 {
match NonZeroU64::new(v) {
Some(v) => v,
None => panic!("span encoding is zero"),
}
}
/// A value with a span locating it in the source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
pub struct Spanned<T> {
@ -39,91 +109,6 @@ impl<T: Debug> Debug for Spanned<T> {
}
}
/// A unique identifier for a syntax node.
///
/// This is used throughout the compiler to track which source section an error
/// or element stems from. Can be [mapped back](super::Source::range) to a byte
/// range for user facing display.
///
/// Span ids are ordered in the tree to enable quickly finding the node with
/// some id:
/// - The id of a parent is always smaller than the ids of any of its children.
/// - The id of a node is always greater than any id in the subtrees of any left
/// sibling and smaller than any id in the subtrees of any right sibling.
///
/// The internal ids of spans stay mostly stable, even for nodes behind an
/// insertion. This is not true for simple ranges as they would shift. Spans can
/// be used as inputs to memoized functions without hurting cache performance
/// when text is inserted somewhere in the document other than the end.
///
/// This type takes up 8 bytes and is null-optimized (i.e. `Option<Span>` also
/// takes 8 bytes).
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub struct Span(NonZeroU64);
impl Span {
// Data layout:
// | 16 bits source id | 48 bits number |
// Number of bits for and minimum and maximum numbers assignable to spans.
const BITS: usize = 48;
const DETACHED: u64 = 1;
/// The full range of numbers available to spans.
pub const FULL: Range<u64> = 2..(1 << Self::BITS);
/// Create a new span from a source id and a unique number.
///
/// Panics if the `number` is not contained in `FULL`.
pub const fn new(id: SourceId, number: u64) -> Self {
assert!(
Self::FULL.start <= number && number < Self::FULL.end,
"span number outside valid range"
);
let bits = ((id.into_u16() as u64) << Self::BITS) | number;
Self(to_non_zero(bits))
}
/// A span that does not point into any source file.
pub const fn detached() -> Self {
Self(to_non_zero(Self::DETACHED))
}
/// The id of the source file the span points into.
pub const fn source(self) -> SourceId {
SourceId::from_u16((self.0.get() >> Self::BITS) as u16)
}
/// The unique number of the span within the source file.
pub const fn number(self) -> u64 {
self.0.get() & ((1 << Self::BITS) - 1)
}
}
/// Convert to a non zero u64.
const fn to_non_zero(v: u64) -> NonZeroU64 {
match NonZeroU64::new(v) {
Some(v) => v,
None => panic!("span encoding is zero"),
}
}
/// Result of numbering a node within an interval.
pub(super) type NumberingResult = Result<(), Unnumberable>;
/// Indicates that a node cannot be numbered within a given interval.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub(super) struct Unnumberable;
impl Display for Unnumberable {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
f.pad("cannot number within this interval")
}
}
impl std::error::Error for Unnumberable {}
#[cfg(test)]
mod tests {
use super::{SourceId, Span};

317
src/syntax/tokens.rs
View File

@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID;
use unscanny::Scanner;
use super::resolve::{resolve_hex, resolve_raw, resolve_string};
use super::{ErrorPos, NodeKind, RawFields, Unit};
use super::{ErrorPos, RawFields, SyntaxKind, Unit};
use crate::geom::{AbsUnit, AngleUnit};
use crate::util::{format_eco, EcoString};
@ -96,7 +96,7 @@ impl<'s> Tokens<'s> {
}
impl<'s> Iterator for Tokens<'s> {
type Item = NodeKind;
type Item = SyntaxKind;
/// Parse the next token in the source code.
#[inline]
@ -107,9 +107,10 @@ impl<'s> Iterator for Tokens<'s> {
// Trivia.
'/' if self.s.eat_if('/') => self.line_comment(),
'/' if self.s.eat_if('*') => self.block_comment(),
'*' if self.s.eat_if('/') => {
NodeKind::Error(ErrorPos::Full, "unexpected end of block comment".into())
}
'*' if self.s.eat_if('/') => SyntaxKind::Error(
ErrorPos::Full,
"unexpected end of block comment".into(),
),
c if c.is_whitespace() => self.whitespace(c),
// Other things.
@ -123,15 +124,15 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
fn line_comment(&mut self) -> NodeKind {
fn line_comment(&mut self) -> SyntaxKind {
self.s.eat_until(is_newline);
if self.s.peek().is_none() {
self.terminated = false;
}
NodeKind::LineComment
SyntaxKind::LineComment
}
fn block_comment(&mut self) -> NodeKind {
fn block_comment(&mut self) -> SyntaxKind {
let mut state = '_';
let mut depth = 1;
self.terminated = false;
@ -159,12 +160,12 @@ impl<'s> Tokens<'s> {
}
}
NodeKind::BlockComment
SyntaxKind::BlockComment
}
fn whitespace(&mut self, c: char) -> NodeKind {
fn whitespace(&mut self, c: char) -> SyntaxKind {
if c == ' ' && !self.s.at(char::is_whitespace) {
return NodeKind::Space { newlines: 0 };
return SyntaxKind::Space { newlines: 0 };
}
self.s.uneat();
@ -185,21 +186,21 @@ impl<'s> Tokens<'s> {
}
}
NodeKind::Space { newlines }
SyntaxKind::Space { newlines }
}
#[inline]
fn markup(&mut self, start: usize, c: char) -> NodeKind {
fn markup(&mut self, start: usize, c: char) -> SyntaxKind {
match c {
// Blocks.
'{' => NodeKind::LeftBrace,
'}' => NodeKind::RightBrace,
'[' => NodeKind::LeftBracket,
']' => NodeKind::RightBracket,
'{' => SyntaxKind::LeftBrace,
'}' => SyntaxKind::RightBrace,
'[' => SyntaxKind::LeftBracket,
']' => SyntaxKind::RightBracket,
// Multi-char things.
'#' => self.hash(start),
'.' if self.s.eat_if("..") => NodeKind::Shorthand('\u{2026}'),
'.' if self.s.eat_if("..") => SyntaxKind::Shorthand('\u{2026}'),
'-' => self.hyph(),
'h' if self.s.eat_if("ttp://") || self.s.eat_if("ttps://") => {
self.link(start)
@ -213,16 +214,16 @@ impl<'s> Tokens<'s> {
'\\' => self.backslash(),
// Single-char things.
'~' => NodeKind::Shorthand('\u{00A0}'),
'\'' => NodeKind::SmartQuote { double: false },
'"' => NodeKind::SmartQuote { double: true },
'*' if !self.in_word() => NodeKind::Star,
'_' if !self.in_word() => NodeKind::Underscore,
'$' => NodeKind::Dollar,
'=' => NodeKind::Eq,
'+' => NodeKind::Plus,
'/' => NodeKind::Slash,
':' => NodeKind::Colon,
'~' => SyntaxKind::Shorthand('\u{00A0}'),
'\'' => SyntaxKind::SmartQuote { double: false },
'"' => SyntaxKind::SmartQuote { double: true },
'*' if !self.in_word() => SyntaxKind::Star,
'_' if !self.in_word() => SyntaxKind::Underscore,
'$' => SyntaxKind::Dollar,
'=' => SyntaxKind::Eq,
'+' => SyntaxKind::Plus,
'/' => SyntaxKind::Slash,
':' => SyntaxKind::Colon,
// Plain text.
_ => self.text(start),
@ -230,7 +231,7 @@ impl<'s> Tokens<'s> {
}
#[inline]
fn text(&mut self, start: usize) -> NodeKind {
fn text(&mut self, start: usize) -> SyntaxKind {
macro_rules! table {
($(|$c:literal)*) => {{
let mut t = [false; 128];
@ -266,67 +267,67 @@ impl<'s> Tokens<'s> {
self.s = s;
}
NodeKind::Text(self.s.from(start).into())
SyntaxKind::Text(self.s.from(start).into())
}
fn backslash(&mut self) -> NodeKind {
fn backslash(&mut self) -> SyntaxKind {
match self.s.peek() {
Some('u') if self.s.eat_if("u{") => {
let sequence = self.s.eat_while(char::is_ascii_alphanumeric);
if self.s.eat_if('}') {
if let Some(c) = resolve_hex(sequence) {
NodeKind::Escape(c)
SyntaxKind::Escape(c)
} else {
NodeKind::Error(
SyntaxKind::Error(
ErrorPos::Full,
"invalid unicode escape sequence".into(),
)
}
} else {
self.terminated = false;
NodeKind::Error(ErrorPos::End, "expected closing brace".into())
SyntaxKind::Error(ErrorPos::End, "expected closing brace".into())
}
}
// Linebreaks.
Some(c) if c.is_whitespace() => NodeKind::Linebreak,
None => NodeKind::Linebreak,
Some(c) if c.is_whitespace() => SyntaxKind::Linebreak,
None => SyntaxKind::Linebreak,
// Escapes.
Some(c) => {
self.s.expect(c);
NodeKind::Escape(c)
SyntaxKind::Escape(c)
}
}
}
fn hash(&mut self, start: usize) -> NodeKind {
fn hash(&mut self, start: usize) -> SyntaxKind {
if self.s.at(is_id_start) {
let read = self.s.eat_while(is_id_continue);
match keyword(read) {
Some(keyword) => keyword,
None => NodeKind::Ident(read.into()),
None => SyntaxKind::Ident(read.into()),
}
} else {
self.text(start)
}
}
fn hyph(&mut self) -> NodeKind {
fn hyph(&mut self) -> SyntaxKind {
if self.s.eat_if('-') {
if self.s.eat_if('-') {
NodeKind::Shorthand('\u{2014}')
SyntaxKind::Shorthand('\u{2014}')
} else {
NodeKind::Shorthand('\u{2013}')
SyntaxKind::Shorthand('\u{2013}')
}
} else if self.s.eat_if('?') {
NodeKind::Shorthand('\u{00AD}')
SyntaxKind::Shorthand('\u{00AD}')
} else {
NodeKind::Minus
SyntaxKind::Minus
}
}
fn link(&mut self, start: usize) -> NodeKind {
fn link(&mut self, start: usize) -> SyntaxKind {
#[rustfmt::skip]
self.s.eat_while(|c: char| matches!(c,
| '0' ..= '9'
@ -338,10 +339,10 @@ impl<'s> Tokens<'s> {
if self.s.scout(-1) == Some('.') {
self.s.uneat();
}
NodeKind::Link(self.s.from(start).into())
SyntaxKind::Link(self.s.from(start).into())
}
fn raw(&mut self) -> NodeKind {
fn raw(&mut self) -> SyntaxKind {
let column = self.column(self.s.cursor() - 1);
let mut backticks = 1;
@ -351,7 +352,7 @@ impl<'s> Tokens<'s> {
// Special case for empty inline block.
if backticks == 2 {
return NodeKind::Raw(Arc::new(RawFields {
return SyntaxKind::Raw(Arc::new(RawFields {
text: EcoString::new(),
lang: None,
block: false,
@ -370,7 +371,7 @@ impl<'s> Tokens<'s> {
if found == backticks {
let end = self.s.cursor() - found as usize;
NodeKind::Raw(Arc::new(resolve_raw(
SyntaxKind::Raw(Arc::new(resolve_raw(
column,
backticks,
self.s.get(start..end),
@ -379,7 +380,7 @@ impl<'s> Tokens<'s> {
self.terminated = false;
let remaining = backticks - found;
let noun = if remaining == 1 { "backtick" } else { "backticks" };
NodeKind::Error(
SyntaxKind::Error(
ErrorPos::End,
if found == 0 {
format_eco!("expected {} {}", remaining, noun)
@ -390,114 +391,114 @@ impl<'s> Tokens<'s> {
}
}
fn numbering(&mut self, start: usize) -> NodeKind {
fn numbering(&mut self, start: usize) -> SyntaxKind {
self.s.eat_while(char::is_ascii_digit);
let read = self.s.from(start);
if self.s.eat_if('.') {
if let Ok(number) = read.parse() {
return NodeKind::EnumNumbering(number);
return SyntaxKind::EnumNumbering(number);
}
}
self.text(start)
}
fn label(&mut self) -> NodeKind {
fn label(&mut self) -> SyntaxKind {
let label = self.s.eat_while(is_id_continue);
if self.s.eat_if('>') {
if !label.is_empty() {
NodeKind::Label(label.into())
SyntaxKind::Label(label.into())
} else {
NodeKind::Error(ErrorPos::Full, "label cannot be empty".into())
SyntaxKind::Error(ErrorPos::Full, "label cannot be empty".into())
}
} else {
self.terminated = false;
NodeKind::Error(ErrorPos::End, "expected closing angle bracket".into())
SyntaxKind::Error(ErrorPos::End, "expected closing angle bracket".into())
}
}
fn reference(&mut self, start: usize) -> NodeKind {
fn reference(&mut self, start: usize) -> SyntaxKind {
let label = self.s.eat_while(is_id_continue);
if !label.is_empty() {
NodeKind::Ref(label.into())
SyntaxKind::Ref(label.into())
} else {
self.text(start)
}
}
fn math(&mut self, start: usize, c: char) -> NodeKind {
fn math(&mut self, start: usize, c: char) -> SyntaxKind {
match c {
// Escape sequences.
'\\' => self.backslash(),
// Single-char things.
'_' => NodeKind::Underscore,
'^' => NodeKind::Hat,
'/' => NodeKind::Slash,
'&' => NodeKind::Amp,
'$' => NodeKind::Dollar,
'_' => SyntaxKind::Underscore,
'^' => SyntaxKind::Hat,
'/' => SyntaxKind::Slash,
'&' => SyntaxKind::Amp,
'$' => SyntaxKind::Dollar,
// Brackets.
'{' => NodeKind::LeftBrace,
'}' => NodeKind::RightBrace,
'[' => NodeKind::LeftBracket,
']' => NodeKind::RightBracket,
'(' => NodeKind::LeftParen,
')' => NodeKind::RightParen,
'{' => SyntaxKind::LeftBrace,
'}' => SyntaxKind::RightBrace,
'[' => SyntaxKind::LeftBracket,
']' => SyntaxKind::RightBracket,
'(' => SyntaxKind::LeftParen,
')' => SyntaxKind::RightParen,
// Identifiers.
c if is_math_id_start(c) && self.s.at(is_math_id_continue) => {
self.s.eat_while(is_math_id_continue);
NodeKind::Ident(self.s.from(start).into())
SyntaxKind::Ident(self.s.from(start).into())
}
// Numbers.
c if c.is_numeric() => {
self.s.eat_while(char::is_numeric);
NodeKind::Atom(self.s.from(start).into())
SyntaxKind::Atom(self.s.from(start).into())
}
// Other math atoms.
c => NodeKind::Atom(c.into()),
c => SyntaxKind::Atom(c.into()),
}
}
fn code(&mut self, start: usize, c: char) -> NodeKind {
fn code(&mut self, start: usize, c: char) -> SyntaxKind {
match c {
// Blocks.
'{' => NodeKind::LeftBrace,
'}' => NodeKind::RightBrace,
'[' => NodeKind::LeftBracket,
']' => NodeKind::RightBracket,
'{' => SyntaxKind::LeftBrace,
'}' => SyntaxKind::RightBrace,
'[' => SyntaxKind::LeftBracket,
']' => SyntaxKind::RightBracket,
// Parentheses.
'(' => NodeKind::LeftParen,
')' => NodeKind::RightParen,
'(' => SyntaxKind::LeftParen,
')' => SyntaxKind::RightParen,
// Two-char operators.
'=' if self.s.eat_if('=') => NodeKind::EqEq,
'!' if self.s.eat_if('=') => NodeKind::ExclEq,
'<' if self.s.eat_if('=') => NodeKind::LtEq,
'>' if self.s.eat_if('=') => NodeKind::GtEq,
'+' if self.s.eat_if('=') => NodeKind::PlusEq,
'-' if self.s.eat_if('=') => NodeKind::HyphEq,
'*' if self.s.eat_if('=') => NodeKind::StarEq,
'/' if self.s.eat_if('=') => NodeKind::SlashEq,
'.' if self.s.eat_if('.') => NodeKind::Dots,
'=' if self.s.eat_if('>') => NodeKind::Arrow,
'=' if self.s.eat_if('=') => SyntaxKind::EqEq,
'!' if self.s.eat_if('=') => SyntaxKind::ExclEq,
'<' if self.s.eat_if('=') => SyntaxKind::LtEq,
'>' if self.s.eat_if('=') => SyntaxKind::GtEq,
'+' if self.s.eat_if('=') => SyntaxKind::PlusEq,
'-' if self.s.eat_if('=') => SyntaxKind::HyphEq,
'*' if self.s.eat_if('=') => SyntaxKind::StarEq,
'/' if self.s.eat_if('=') => SyntaxKind::SlashEq,
'.' if self.s.eat_if('.') => SyntaxKind::Dots,
'=' if self.s.eat_if('>') => SyntaxKind::Arrow,
// Single-char operators.
',' => NodeKind::Comma,
';' => NodeKind::Semicolon,
':' => NodeKind::Colon,
'+' => NodeKind::Plus,
'-' => NodeKind::Minus,
'*' => NodeKind::Star,
'/' => NodeKind::Slash,
'=' => NodeKind::Eq,
'<' => NodeKind::Lt,
'>' => NodeKind::Gt,
'.' if !self.s.at(char::is_ascii_digit) => NodeKind::Dot,
',' => SyntaxKind::Comma,
';' => SyntaxKind::Semicolon,
':' => SyntaxKind::Colon,
'+' => SyntaxKind::Plus,
'-' => SyntaxKind::Minus,
'*' => SyntaxKind::Star,
'/' => SyntaxKind::Slash,
'=' => SyntaxKind::Eq,
'<' => SyntaxKind::Lt,
'>' => SyntaxKind::Gt,
'.' if !self.s.at(char::is_ascii_digit) => SyntaxKind::Dot,
// Identifiers.
c if is_id_start(c) => self.ident(start),
@ -511,22 +512,22 @@ impl<'s> Tokens<'s> {
'"' => self.string(),
// Invalid token.
_ => NodeKind::Error(ErrorPos::Full, "not valid here".into()),
_ => SyntaxKind::Error(ErrorPos::Full, "not valid here".into()),
}
}
fn ident(&mut self, start: usize) -> NodeKind {
fn ident(&mut self, start: usize) -> SyntaxKind {
self.s.eat_while(is_id_continue);
match self.s.from(start) {
"none" => NodeKind::None,
"auto" => NodeKind::Auto,
"true" => NodeKind::Bool(true),
"false" => NodeKind::Bool(false),
id => keyword(id).unwrap_or_else(|| NodeKind::Ident(id.into())),
"none" => SyntaxKind::None,
"auto" => SyntaxKind::Auto,
"true" => SyntaxKind::Bool(true),
"false" => SyntaxKind::Bool(false),
id => keyword(id).unwrap_or_else(|| SyntaxKind::Ident(id.into())),
}
}
fn number(&mut self, start: usize, c: char) -> NodeKind {
fn number(&mut self, start: usize, c: char) -> SyntaxKind {
// Read the first part (integer or fractional depending on `first`).
self.s.eat_while(char::is_ascii_digit);
@ -554,30 +555,30 @@ impl<'s> Tokens<'s> {
// Find out whether it is a simple number.
if suffix.is_empty() {
if let Ok(i) = number.parse::<i64>() {
return NodeKind::Int(i);
return SyntaxKind::Int(i);
}
}
let Ok(v) = number.parse::<f64>() else {
return NodeKind::Error(ErrorPos::Full, "invalid number".into());
return SyntaxKind::Error(ErrorPos::Full, "invalid number".into());
};
match suffix {
"" => NodeKind::Float(v),
"pt" => NodeKind::Numeric(v, Unit::Length(AbsUnit::Pt)),
"mm" => NodeKind::Numeric(v, Unit::Length(AbsUnit::Mm)),
"cm" => NodeKind::Numeric(v, Unit::Length(AbsUnit::Cm)),
"in" => NodeKind::Numeric(v, Unit::Length(AbsUnit::In)),
"deg" => NodeKind::Numeric(v, Unit::Angle(AngleUnit::Deg)),
"rad" => NodeKind::Numeric(v, Unit::Angle(AngleUnit::Rad)),
"em" => NodeKind::Numeric(v, Unit::Em),
"fr" => NodeKind::Numeric(v, Unit::Fr),
"%" => NodeKind::Numeric(v, Unit::Percent),
_ => NodeKind::Error(ErrorPos::Full, "invalid number suffix".into()),
"" => SyntaxKind::Float(v),
"pt" => SyntaxKind::Numeric(v, Unit::Length(AbsUnit::Pt)),
"mm" => SyntaxKind::Numeric(v, Unit::Length(AbsUnit::Mm)),
"cm" => SyntaxKind::Numeric(v, Unit::Length(AbsUnit::Cm)),
"in" => SyntaxKind::Numeric(v, Unit::Length(AbsUnit::In)),
"deg" => SyntaxKind::Numeric(v, Unit::Angle(AngleUnit::Deg)),
"rad" => SyntaxKind::Numeric(v, Unit::Angle(AngleUnit::Rad)),
"em" => SyntaxKind::Numeric(v, Unit::Em),
"fr" => SyntaxKind::Numeric(v, Unit::Fr),
"%" => SyntaxKind::Numeric(v, Unit::Percent),
_ => SyntaxKind::Error(ErrorPos::Full, "invalid number suffix".into()),
}
}
fn string(&mut self) -> NodeKind {
fn string(&mut self) -> SyntaxKind {
let mut escaped = false;
let verbatim = self.s.eat_until(|c| {
if c == '"' && !escaped {
@ -590,10 +591,10 @@ impl<'s> Tokens<'s> {
let string = resolve_string(verbatim);
if self.s.eat_if('"') {
NodeKind::Str(string)
SyntaxKind::Str(string)
} else {
self.terminated = false;
NodeKind::Error(ErrorPos::End, "expected quote".into())
SyntaxKind::Error(ErrorPos::End, "expected quote".into())
}
}
@ -605,25 +606,25 @@ impl<'s> Tokens<'s> {
}
}
fn keyword(ident: &str) -> Option<NodeKind> {
fn keyword(ident: &str) -> Option<SyntaxKind> {
Some(match ident {
"not" => NodeKind::Not,
"and" => NodeKind::And,
"or" => NodeKind::Or,
"let" => NodeKind::Let,
"set" => NodeKind::Set,
"show" => NodeKind::Show,
"if" => NodeKind::If,
"else" => NodeKind::Else,
"for" => NodeKind::For,
"in" => NodeKind::In,
"while" => NodeKind::While,
"break" => NodeKind::Break,
"continue" => NodeKind::Continue,
"return" => NodeKind::Return,
"import" => NodeKind::Import,
"include" => NodeKind::Include,
"from" => NodeKind::From,
"not" => SyntaxKind::Not,
"and" => SyntaxKind::And,
"or" => SyntaxKind::Or,
"let" => SyntaxKind::Let,
"set" => SyntaxKind::Set,
"show" => SyntaxKind::Show,
"if" => SyntaxKind::If,
"else" => SyntaxKind::Else,
"for" => SyntaxKind::For,
"in" => SyntaxKind::In,
"while" => SyntaxKind::While,
"break" => SyntaxKind::Break,
"continue" => SyntaxKind::Continue,
"return" => SyntaxKind::Return,
"import" => SyntaxKind::Import,
"include" => SyntaxKind::Include,
"from" => SyntaxKind::From,
_ => return None,
})
}
@ -715,36 +716,36 @@ mod tests {
use super::*;
use ErrorPos::*;
use NodeKind::*;
use Option::None;
use SyntaxKind::*;
use TokenMode::{Code, Markup};
fn Space(newlines: usize) -> NodeKind {
NodeKind::Space { newlines }
fn Space(newlines: usize) -> SyntaxKind {
SyntaxKind::Space { newlines }
}
fn Raw(text: &str, lang: Option<&str>, block: bool) -> NodeKind {
NodeKind::Raw(Arc::new(RawFields {
fn Raw(text: &str, lang: Option<&str>, block: bool) -> SyntaxKind {
SyntaxKind::Raw(Arc::new(RawFields {
text: text.into(),
lang: lang.map(Into::into),
block,
}))
}
fn Str(string: &str) -> NodeKind {
NodeKind::Str(string.into())
fn Str(string: &str) -> SyntaxKind {
SyntaxKind::Str(string.into())
}
fn Text(string: &str) -> NodeKind {
NodeKind::Text(string.into())
fn Text(string: &str) -> SyntaxKind {
SyntaxKind::Text(string.into())
}
fn Ident(ident: &str) -> NodeKind {
NodeKind::Ident(ident.into())
fn Ident(ident: &str) -> SyntaxKind {
SyntaxKind::Ident(ident.into())
}
fn Error(pos: ErrorPos, message: &str) -> NodeKind {
NodeKind::Error(pos, message.into())
fn Error(pos: ErrorPos, message: &str) -> SyntaxKind {
SyntaxKind::Error(pos, message.into())
}
/// Building blocks for suffix testing.
@ -769,7 +770,7 @@ mod tests {
// - the suffix string
// - the resulting suffix NodeKind
fn suffixes(
) -> impl Iterator<Item = (char, Option<TokenMode>, &'static str, NodeKind)> {
) -> impl Iterator<Item = (char, Option<TokenMode>, &'static str, SyntaxKind)> {
[
// Whitespace suffixes.
(' ', None, " ", Space(0)),
@ -1089,7 +1090,7 @@ mod tests {
// Combined integers and floats.
let nums = ints.iter().map(|&(k, v)| (k, v as f64)).chain(floats);
let suffixes: &[(&str, fn(f64) -> NodeKind)] = &[
let suffixes: &[(&str, fn(f64) -> SyntaxKind)] = &[
("mm", |x| Numeric(x, Unit::Length(AbsUnit::Mm))),
("pt", |x| Numeric(x, Unit::Length(AbsUnit::Pt))),
("cm", |x| Numeric(x, Unit::Length(AbsUnit::Cm))),