Rewrite parser
This commit is contained in:
Laurenz
parent
40561e57fb
commit
dd331f007c
@ -141,7 +141,7 @@ fn complete_params(ctx: &mut CompletionContext) -> bool {
|
||||
if let Some(param) = before_colon.cast::<ast::Ident>();
|
||||
then {
|
||||
ctx.from = match ctx.leaf.kind() {
|
||||
SyntaxKind::Colon | SyntaxKind::Space { .. } => ctx.cursor,
|
||||
SyntaxKind::Colon | SyntaxKind::Space => ctx.cursor,
|
||||
_ => ctx.leaf.offset(),
|
||||
};
|
||||
ctx.named_param_value_completions(&callee, ¶m);
|
||||
@ -246,7 +246,7 @@ fn complete_symbols(ctx: &mut CompletionContext) -> bool {
|
||||
/// Complete in markup mode.
|
||||
fn complete_markup(ctx: &mut CompletionContext) -> bool {
|
||||
// Bail if we aren't even in markup.
|
||||
if !matches!(ctx.leaf.parent_kind(), None | Some(SyntaxKind::Markup { .. })) {
|
||||
if !matches!(ctx.leaf.parent_kind(), None | Some(SyntaxKind::Markup)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -325,7 +325,7 @@ fn complete_math(ctx: &mut CompletionContext) -> bool {
|
||||
fn complete_code(ctx: &mut CompletionContext) -> bool {
|
||||
if matches!(
|
||||
ctx.leaf.parent_kind(),
|
||||
None | Some(SyntaxKind::Markup { .. }) | Some(SyntaxKind::Math)
|
||||
None | Some(SyntaxKind::Markup) | Some(SyntaxKind::Math)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
@ -887,7 +887,7 @@ impl<'a> CompletionContext<'a> {
|
||||
|
||||
self.snippet_completion(
|
||||
"import",
|
||||
"import ${items} from \"${file.typ}\"",
|
||||
"import \"${file.typ}\": ${items}",
|
||||
"Imports variables from another file.",
|
||||
);
|
||||
|
||||
|
||||
@ -83,9 +83,41 @@ impl Category {
|
||||
/// highlighted.
|
||||
pub fn highlight(node: &LinkedNode) -> Option<Category> {
|
||||
match node.kind() {
|
||||
SyntaxKind::LineComment => Some(Category::Comment),
|
||||
SyntaxKind::BlockComment => Some(Category::Comment),
|
||||
SyntaxKind::Space { .. } => None,
|
||||
SyntaxKind::Markup
|
||||
if node.parent_kind() == Some(SyntaxKind::TermItem)
|
||||
&& node.next_sibling().as_ref().map(|v| v.kind())
|
||||
== Some(SyntaxKind::Colon) =>
|
||||
{
|
||||
Some(Category::ListTerm)
|
||||
}
|
||||
SyntaxKind::Markup => None,
|
||||
SyntaxKind::Text => None,
|
||||
SyntaxKind::Space => None,
|
||||
SyntaxKind::Linebreak => Some(Category::Escape),
|
||||
SyntaxKind::Parbreak => None,
|
||||
SyntaxKind::Escape => Some(Category::Escape),
|
||||
SyntaxKind::Shorthand => Some(Category::Escape),
|
||||
SyntaxKind::Symbol => Some(Category::Escape),
|
||||
SyntaxKind::SmartQuote => None,
|
||||
SyntaxKind::Strong => Some(Category::Strong),
|
||||
SyntaxKind::Emph => Some(Category::Emph),
|
||||
SyntaxKind::Raw => Some(Category::Raw),
|
||||
SyntaxKind::Link => Some(Category::Link),
|
||||
SyntaxKind::Label => Some(Category::Label),
|
||||
SyntaxKind::Ref => Some(Category::Ref),
|
||||
SyntaxKind::Heading => Some(Category::Heading),
|
||||
SyntaxKind::HeadingMarker => None,
|
||||
SyntaxKind::ListItem => None,
|
||||
SyntaxKind::ListMarker => Some(Category::ListMarker),
|
||||
SyntaxKind::EnumItem => None,
|
||||
SyntaxKind::EnumMarker => Some(Category::ListMarker),
|
||||
SyntaxKind::TermItem => None,
|
||||
SyntaxKind::TermMarker => Some(Category::ListMarker),
|
||||
SyntaxKind::Math => None,
|
||||
SyntaxKind::Atom => None,
|
||||
SyntaxKind::Script => None,
|
||||
SyntaxKind::Frac => None,
|
||||
SyntaxKind::AlignPoint => Some(Category::MathOperator),
|
||||
|
||||
SyntaxKind::LeftBrace => Some(Category::Punctuation),
|
||||
SyntaxKind::RightBrace => Some(Category::Punctuation),
|
||||
@ -105,16 +137,9 @@ pub fn highlight(node: &LinkedNode) -> Option<Category> {
|
||||
_ => None,
|
||||
},
|
||||
SyntaxKind::Dollar => Some(Category::MathDelimiter),
|
||||
SyntaxKind::Plus => Some(match node.parent_kind() {
|
||||
Some(SyntaxKind::EnumItem) => Category::ListMarker,
|
||||
_ => Category::Operator,
|
||||
}),
|
||||
SyntaxKind::Minus => Some(match node.parent_kind() {
|
||||
Some(SyntaxKind::ListItem) => Category::ListMarker,
|
||||
_ => Category::Operator,
|
||||
}),
|
||||
SyntaxKind::Plus => Some(Category::Operator),
|
||||
SyntaxKind::Minus => Some(Category::Operator),
|
||||
SyntaxKind::Slash => Some(match node.parent_kind() {
|
||||
Some(SyntaxKind::TermItem) => Category::ListMarker,
|
||||
Some(SyntaxKind::Frac) => Category::MathOperator,
|
||||
_ => Category::Operator,
|
||||
}),
|
||||
@ -157,41 +182,9 @@ pub fn highlight(node: &LinkedNode) -> Option<Category> {
|
||||
SyntaxKind::Include => Some(Category::Keyword),
|
||||
SyntaxKind::As => Some(Category::Keyword),
|
||||
|
||||
SyntaxKind::Markup { .. }
|
||||
if node.parent_kind() == Some(SyntaxKind::TermItem)
|
||||
&& node.next_sibling().as_ref().map(|v| v.kind())
|
||||
== Some(SyntaxKind::Colon) =>
|
||||
{
|
||||
Some(Category::ListTerm)
|
||||
}
|
||||
SyntaxKind::Markup { .. } => None,
|
||||
|
||||
SyntaxKind::Text => None,
|
||||
SyntaxKind::Linebreak => Some(Category::Escape),
|
||||
SyntaxKind::Escape => Some(Category::Escape),
|
||||
SyntaxKind::Shorthand => Some(Category::Escape),
|
||||
SyntaxKind::Symbol => Some(Category::Escape),
|
||||
SyntaxKind::SmartQuote { .. } => None,
|
||||
SyntaxKind::Strong => Some(Category::Strong),
|
||||
SyntaxKind::Emph => Some(Category::Emph),
|
||||
SyntaxKind::Raw { .. } => Some(Category::Raw),
|
||||
SyntaxKind::Link => Some(Category::Link),
|
||||
SyntaxKind::Label => Some(Category::Label),
|
||||
SyntaxKind::Ref => Some(Category::Ref),
|
||||
SyntaxKind::Heading => Some(Category::Heading),
|
||||
SyntaxKind::ListItem => None,
|
||||
SyntaxKind::EnumItem => None,
|
||||
SyntaxKind::EnumNumbering => Some(Category::ListMarker),
|
||||
SyntaxKind::TermItem => None,
|
||||
SyntaxKind::Math => None,
|
||||
SyntaxKind::Atom => None,
|
||||
SyntaxKind::Script => None,
|
||||
SyntaxKind::Frac => None,
|
||||
SyntaxKind::AlignPoint => Some(Category::MathOperator),
|
||||
|
||||
SyntaxKind::Ident => match node.parent_kind() {
|
||||
Some(
|
||||
SyntaxKind::Markup { .. }
|
||||
SyntaxKind::Markup
|
||||
| SyntaxKind::Math
|
||||
| SyntaxKind::Script
|
||||
| SyntaxKind::Frac,
|
||||
@ -258,7 +251,10 @@ pub fn highlight(node: &LinkedNode) -> Option<Category> {
|
||||
SyntaxKind::LoopContinue => None,
|
||||
SyntaxKind::FuncReturn => None,
|
||||
|
||||
SyntaxKind::LineComment => Some(Category::Comment),
|
||||
SyntaxKind::BlockComment => Some(Category::Comment),
|
||||
SyntaxKind::Error => Some(Category::Error),
|
||||
SyntaxKind::Eof => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -261,9 +261,10 @@ impl Eval for ast::Expr {
|
||||
};
|
||||
|
||||
match self {
|
||||
Self::Text(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Space(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Linebreak(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Text(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Parbreak(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Escape(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Shorthand(v) => v.eval(vm).map(Value::Content),
|
||||
Self::Symbol(v) => v.eval(vm).map(Value::Content),
|
||||
@ -330,14 +331,19 @@ impl ast::Expr {
|
||||
}
|
||||
}
|
||||
|
||||
impl Eval for ast::Text {
|
||||
type Output = Content;
|
||||
|
||||
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
|
||||
Ok((vm.items.text)(self.get().clone()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Eval for ast::Space {
|
||||
type Output = Content;
|
||||
|
||||
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
|
||||
Ok(match self.newlines() {
|
||||
0..=1 => (vm.items.space)(),
|
||||
_ => (vm.items.parbreak)(),
|
||||
})
|
||||
Ok((vm.items.space)())
|
||||
}
|
||||
}
|
||||
|
||||
@ -349,11 +355,11 @@ impl Eval for ast::Linebreak {
|
||||
}
|
||||
}
|
||||
|
||||
impl Eval for ast::Text {
|
||||
impl Eval for ast::Parbreak {
|
||||
type Output = Content;
|
||||
|
||||
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
|
||||
Ok((vm.items.text)(self.get().clone()))
|
||||
Ok((vm.items.parbreak)())
|
||||
}
|
||||
}
|
||||
|
||||
@ -438,7 +444,7 @@ impl Eval for ast::Link {
|
||||
type Output = Content;
|
||||
|
||||
fn eval(&self, vm: &mut Vm) -> SourceResult<Self::Output> {
|
||||
Ok((vm.items.link)(self.url().clone()))
|
||||
Ok((vm.items.link)(self.get().clone()))
|
||||
}
|
||||
}
|
||||
|
||||
@ -1231,13 +1237,17 @@ impl Eval for ast::ModuleImport {
|
||||
}
|
||||
}
|
||||
Some(ast::Imports::Items(idents)) => {
|
||||
let mut errors = vec![];
|
||||
for ident in idents {
|
||||
if let Some(value) = module.scope().get(&ident) {
|
||||
vm.scopes.top.define(ident.take(), value.clone());
|
||||
} else {
|
||||
bail!(ident.span(), "unresolved import");
|
||||
errors.push(error!(ident.span(), "unresolved import"));
|
||||
}
|
||||
}
|
||||
if errors.len() > 0 {
|
||||
return Err(Box::new(errors));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -29,9 +29,6 @@ pub trait AstNode: Sized {
|
||||
|
||||
macro_rules! node {
|
||||
($(#[$attr:meta])* $name:ident) => {
|
||||
node!{ $(#[$attr])* $name: SyntaxKind::$name { .. } }
|
||||
};
|
||||
($(#[$attr:meta])* $name:ident: $variants:pat) => {
|
||||
#[derive(Debug, Clone, PartialEq, Hash)]
|
||||
#[repr(transparent)]
|
||||
$(#[$attr])*
|
||||
@ -39,7 +36,7 @@ macro_rules! node {
|
||||
|
||||
impl AstNode for $name {
|
||||
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
|
||||
if matches!(node.kind(), $variants) {
|
||||
if matches!(node.kind(), SyntaxKind::$name) {
|
||||
Some(Self(node.clone()))
|
||||
} else {
|
||||
Option::None
|
||||
@ -67,8 +64,7 @@ impl Markup {
|
||||
.filter(move |node| {
|
||||
// Ignore newline directly after statements without semicolons.
|
||||
let kind = node.kind();
|
||||
let keep =
|
||||
!was_stmt || !matches!(kind, SyntaxKind::Space { newlines: 1 });
|
||||
let keep = !was_stmt || node.kind() != SyntaxKind::Space;
|
||||
was_stmt = kind.is_stmt();
|
||||
keep
|
||||
})
|
||||
@ -79,12 +75,15 @@ impl Markup {
|
||||
/// An expression in markup, math or code.
|
||||
#[derive(Debug, Clone, PartialEq, Hash)]
|
||||
pub enum Expr {
|
||||
/// Whitespace.
|
||||
/// Plain text without markup.
|
||||
Text(Text),
|
||||
/// Whitespace in markup or math. Has at most one newline in markup, as more
|
||||
/// indicate a paragraph break.
|
||||
Space(Space),
|
||||
/// A forced line break: `\`.
|
||||
Linebreak(Linebreak),
|
||||
/// Plain text without markup.
|
||||
Text(Text),
|
||||
/// A paragraph break, indicated by one or multiple blank lines.
|
||||
Parbreak(Parbreak),
|
||||
/// An escape sequence: `\#`, `\u{1F5FA}`.
|
||||
Escape(Escape),
|
||||
/// A shorthand for a unicode codepoint. For example, `~` for non-breaking
|
||||
@ -189,7 +188,7 @@ pub enum Expr {
|
||||
impl Expr {
|
||||
fn cast_with_space(node: &SyntaxNode) -> Option<Self> {
|
||||
match node.kind() {
|
||||
SyntaxKind::Space { .. } => node.cast().map(Self::Space),
|
||||
SyntaxKind::Space => node.cast().map(Self::Space),
|
||||
_ => Self::from_untyped(node),
|
||||
}
|
||||
}
|
||||
@ -199,14 +198,15 @@ impl AstNode for Expr {
|
||||
fn from_untyped(node: &SyntaxNode) -> Option<Self> {
|
||||
match node.kind() {
|
||||
SyntaxKind::Linebreak => node.cast().map(Self::Linebreak),
|
||||
SyntaxKind::Parbreak => node.cast().map(Self::Parbreak),
|
||||
SyntaxKind::Text => node.cast().map(Self::Text),
|
||||
SyntaxKind::Escape => node.cast().map(Self::Escape),
|
||||
SyntaxKind::Shorthand => node.cast().map(Self::Shorthand),
|
||||
SyntaxKind::Symbol => node.cast().map(Self::Symbol),
|
||||
SyntaxKind::SmartQuote { .. } => node.cast().map(Self::SmartQuote),
|
||||
SyntaxKind::SmartQuote => node.cast().map(Self::SmartQuote),
|
||||
SyntaxKind::Strong => node.cast().map(Self::Strong),
|
||||
SyntaxKind::Emph => node.cast().map(Self::Emph),
|
||||
SyntaxKind::Raw { .. } => node.cast().map(Self::Raw),
|
||||
SyntaxKind::Raw => node.cast().map(Self::Raw),
|
||||
SyntaxKind::Link => node.cast().map(Self::Link),
|
||||
SyntaxKind::Label => node.cast().map(Self::Label),
|
||||
SyntaxKind::Ref => node.cast().map(Self::Ref),
|
||||
@ -255,9 +255,10 @@ impl AstNode for Expr {
|
||||
|
||||
fn as_untyped(&self) -> &SyntaxNode {
|
||||
match self {
|
||||
Self::Text(v) => v.as_untyped(),
|
||||
Self::Space(v) => v.as_untyped(),
|
||||
Self::Linebreak(v) => v.as_untyped(),
|
||||
Self::Text(v) => v.as_untyped(),
|
||||
Self::Parbreak(v) => v.as_untyped(),
|
||||
Self::Escape(v) => v.as_untyped(),
|
||||
Self::Shorthand(v) => v.as_untyped(),
|
||||
Self::Symbol(v) => v.as_untyped(),
|
||||
@ -311,26 +312,6 @@ impl AstNode for Expr {
|
||||
}
|
||||
}
|
||||
|
||||
node! {
|
||||
/// Whitespace.
|
||||
Space
|
||||
}
|
||||
|
||||
impl Space {
|
||||
/// Get the number of newlines.
|
||||
pub fn newlines(&self) -> usize {
|
||||
match self.0.kind() {
|
||||
SyntaxKind::Space { newlines } => newlines,
|
||||
_ => panic!("space is of wrong kind"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node! {
|
||||
/// A forced line break: `\`.
|
||||
Linebreak
|
||||
}
|
||||
|
||||
node! {
|
||||
/// Plain text without markup.
|
||||
Text
|
||||
@ -343,6 +324,22 @@ impl Text {
|
||||
}
|
||||
}
|
||||
|
||||
node! {
|
||||
/// Whitespace in markup or math. Has at most one newline in markup, as more
|
||||
/// indicate a paragraph break.
|
||||
Space
|
||||
}
|
||||
|
||||
node! {
|
||||
/// A forced line break: `\`.
|
||||
Linebreak
|
||||
}
|
||||
|
||||
node! {
|
||||
/// A paragraph break, indicated by one or multiple blank lines.
|
||||
Parbreak
|
||||
}
|
||||
|
||||
node! {
|
||||
/// An escape sequence: `\#`, `\u{1F5FA}`.
|
||||
Escape
|
||||
@ -454,10 +451,6 @@ node! {
|
||||
impl Raw {
|
||||
/// The trimmed raw text.
|
||||
pub fn text(&self) -> EcoString {
|
||||
let SyntaxKind::Raw { column } = self.0.kind() else {
|
||||
panic!("raw node is of wrong kind");
|
||||
};
|
||||
|
||||
let mut text = self.0.text().as_str();
|
||||
let blocky = text.starts_with("```");
|
||||
text = text.trim_matches('`');
|
||||
@ -480,14 +473,16 @@ impl Raw {
|
||||
let mut lines = split_newlines(text);
|
||||
|
||||
if blocky {
|
||||
let dedent = lines
|
||||
.iter()
|
||||
.skip(1)
|
||||
.map(|line| line.chars().take_while(|c| c.is_whitespace()).count())
|
||||
.min()
|
||||
.unwrap_or(0);
|
||||
|
||||
// Dedent based on column, but not for the first line.
|
||||
for line in lines.iter_mut().skip(1) {
|
||||
let offset = line
|
||||
.chars()
|
||||
.take(column)
|
||||
.take_while(|c| c.is_whitespace())
|
||||
.map(char::len_utf8)
|
||||
.sum();
|
||||
let offset = line.chars().take(dedent).map(char::len_utf8).sum();
|
||||
*line = &line[offset..];
|
||||
}
|
||||
|
||||
@ -531,7 +526,7 @@ node! {
|
||||
|
||||
impl Link {
|
||||
/// Get the URL.
|
||||
pub fn url(&self) -> &EcoString {
|
||||
pub fn get(&self) -> &EcoString {
|
||||
self.0.text()
|
||||
}
|
||||
}
|
||||
@ -575,10 +570,9 @@ impl Heading {
|
||||
pub fn level(&self) -> NonZeroUsize {
|
||||
self.0
|
||||
.children()
|
||||
.filter(|n| n.kind() == SyntaxKind::Eq)
|
||||
.count()
|
||||
.try_into()
|
||||
.expect("heading is missing equals sign")
|
||||
.find(|node| node.kind() == SyntaxKind::HeadingMarker)
|
||||
.and_then(|node| node.len().try_into().ok())
|
||||
.expect("heading is missing marker")
|
||||
}
|
||||
}
|
||||
|
||||
@ -603,7 +597,7 @@ impl EnumItem {
|
||||
/// The explicit numbering, if any: `23.`.
|
||||
pub fn number(&self) -> Option<NonZeroUsize> {
|
||||
self.0.children().find_map(|node| match node.kind() {
|
||||
SyntaxKind::EnumNumbering => node.text().trim_end_matches('.').parse().ok(),
|
||||
SyntaxKind::EnumMarker => node.text().trim_end_matches('.').parse().ok(),
|
||||
_ => Option::None,
|
||||
})
|
||||
}
|
||||
@ -765,7 +759,7 @@ node! {
|
||||
}
|
||||
|
||||
impl Bool {
|
||||
/// Get the value.
|
||||
/// Get the boolean value.
|
||||
pub fn get(&self) -> bool {
|
||||
self.0.text() == "true"
|
||||
}
|
||||
@ -777,7 +771,7 @@ node! {
|
||||
}
|
||||
|
||||
impl Int {
|
||||
/// Get the value.
|
||||
/// Get the integer value.
|
||||
pub fn get(&self) -> i64 {
|
||||
self.0.text().parse().expect("integer is invalid")
|
||||
}
|
||||
@ -789,7 +783,7 @@ node! {
|
||||
}
|
||||
|
||||
impl Float {
|
||||
/// Get the value.
|
||||
/// Get the floating-point value.
|
||||
pub fn get(&self) -> f64 {
|
||||
self.0.text().parse().expect("float is invalid")
|
||||
}
|
||||
@ -801,7 +795,7 @@ node! {
|
||||
}
|
||||
|
||||
impl Numeric {
|
||||
/// Get the value and unit.
|
||||
/// Get the numeric value and unit.
|
||||
pub fn get(&self) -> (f64, Unit) {
|
||||
let text = self.0.text();
|
||||
let count = text
|
||||
@ -850,7 +844,7 @@ node! {
|
||||
}
|
||||
|
||||
impl Str {
|
||||
/// Get the value.
|
||||
/// Get the string value with resolved escape sequences.
|
||||
pub fn get(&self) -> EcoString {
|
||||
let text = self.0.text();
|
||||
let unquoted = &text[1..text.len() - 1];
|
||||
@ -1058,7 +1052,7 @@ impl Unary {
|
||||
pub fn op(&self) -> UnOp {
|
||||
self.0
|
||||
.children()
|
||||
.find_map(|node| UnOp::from_token(node.kind()))
|
||||
.find_map(|node| UnOp::from_kind(node.kind()))
|
||||
.expect("unary operation is missing operator")
|
||||
}
|
||||
|
||||
@ -1081,7 +1075,7 @@ pub enum UnOp {
|
||||
|
||||
impl UnOp {
|
||||
/// Try to convert the token into a unary operation.
|
||||
pub fn from_token(token: SyntaxKind) -> Option<Self> {
|
||||
pub fn from_kind(token: SyntaxKind) -> Option<Self> {
|
||||
Some(match token {
|
||||
SyntaxKind::Plus => Self::Pos,
|
||||
SyntaxKind::Minus => Self::Neg,
|
||||
@ -1125,7 +1119,7 @@ impl Binary {
|
||||
Option::None
|
||||
}
|
||||
SyntaxKind::In if not => Some(BinOp::NotIn),
|
||||
_ => BinOp::from_token(node.kind()),
|
||||
_ => BinOp::from_kind(node.kind()),
|
||||
})
|
||||
.expect("binary operation is missing operator")
|
||||
}
|
||||
@ -1190,7 +1184,7 @@ pub enum BinOp {
|
||||
|
||||
impl BinOp {
|
||||
/// Try to convert the token into a binary operation.
|
||||
pub fn from_token(token: SyntaxKind) -> Option<Self> {
|
||||
pub fn from_kind(token: SyntaxKind) -> Option<Self> {
|
||||
Some(match token {
|
||||
SyntaxKind::Plus => Self::Add,
|
||||
SyntaxKind::Minus => Self::Sub,
|
||||
|
||||
@ -1,17 +1,72 @@
|
||||
/// All syntactical building blocks that can be part of a Typst document.
|
||||
/// A syntactical building block of a Typst file.
|
||||
///
|
||||
/// Can be created by the lexer or by the parser.
|
||||
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
|
||||
#[repr(u8)]
|
||||
pub enum SyntaxKind {
|
||||
/// A line comment: `// ...`.
|
||||
LineComment,
|
||||
/// A block comment: `/* ... */`.
|
||||
BlockComment,
|
||||
/// One or more whitespace characters. Single spaces are collapsed into text
|
||||
/// nodes if they would otherwise be surrounded by text nodes.
|
||||
/// Markup of which all lines must have a minimal indentation.
|
||||
///
|
||||
/// Also stores how many newlines are contained.
|
||||
Space { newlines: usize },
|
||||
/// Notably, the number does not determine in which column the markup
|
||||
/// started, but to the right of which column all markup elements must be,
|
||||
/// so it is zero except inside indent-aware constructs like lists.
|
||||
Markup,
|
||||
/// Plain text without markup.
|
||||
Text,
|
||||
/// Whitespace. Contains at most one newline in markup, as more indicate a
|
||||
/// paragraph break.
|
||||
Space,
|
||||
/// A forced line break: `\`.
|
||||
Linebreak,
|
||||
/// A paragraph break, indicated by one or multiple blank lines.
|
||||
Parbreak,
|
||||
/// An escape sequence: `\#`, `\u{1F5FA}`.
|
||||
Escape,
|
||||
/// A shorthand for a unicode codepoint. For example, `~` for non-breaking
|
||||
/// space or `-?` for a soft hyphen.
|
||||
Shorthand,
|
||||
/// Symbol notation: `:arrow:l:`. The string only contains the inner part
|
||||
/// without leading and trailing dot.
|
||||
Symbol,
|
||||
/// A smart quote: `'` or `"`.
|
||||
SmartQuote,
|
||||
/// Strong content: `*Strong*`.
|
||||
Strong,
|
||||
/// Emphasized content: `_Emphasized_`.
|
||||
Emph,
|
||||
/// Raw text with optional syntax highlighting: `` `...` ``.
|
||||
Raw,
|
||||
/// A hyperlink: `https://typst.org`.
|
||||
Link,
|
||||
/// A label: `<intro>`.
|
||||
Label,
|
||||
/// A reference: `@target`.
|
||||
Ref,
|
||||
/// A section heading: `= Introduction`.
|
||||
Heading,
|
||||
/// Introduces a section heading: `=`, `==`, ...
|
||||
HeadingMarker,
|
||||
/// An item in a bullet list: `- ...`.
|
||||
ListItem,
|
||||
/// Introduces a list item: `-`.
|
||||
ListMarker,
|
||||
/// An item in an enumeration (numbered list): `+ ...` or `1. ...`.
|
||||
EnumItem,
|
||||
/// Introduces an enumeration item: `+`, `1.`.
|
||||
EnumMarker,
|
||||
/// An item in a term list: `/ Term: Details`.
|
||||
TermItem,
|
||||
/// Introduces a term item: `/`.
|
||||
TermMarker,
|
||||
/// A mathematical formula: `$x$`, `$ x^2 $`.
|
||||
Math,
|
||||
/// An atom in math: `x`, `+`, `12`.
|
||||
Atom,
|
||||
/// A base with optional sub- and superscripts in math: `a_1^2`.
|
||||
Script,
|
||||
/// A fraction in math: `x/2`.
|
||||
Frac,
|
||||
/// An alignment point in math: `&`.
|
||||
AlignPoint,
|
||||
|
||||
/// A left curly brace, starting a code block: `{`.
|
||||
LeftBrace,
|
||||
@ -37,19 +92,17 @@ pub enum SyntaxKind {
|
||||
/// The strong text toggle, multiplication operator, and wildcard import
|
||||
/// symbol: `*`.
|
||||
Star,
|
||||
/// Toggles emphasized text and indicates a subscript in a formula: `_`.
|
||||
/// Toggles emphasized text and indicates a subscript in math: `_`.
|
||||
Underscore,
|
||||
/// Starts and ends a math formula: `$`.
|
||||
Dollar,
|
||||
/// The unary plus, binary addition operator, and start of enum items: `+`.
|
||||
/// The unary plus and binary addition operator: `+`.
|
||||
Plus,
|
||||
/// The unary negation, binary subtraction operator, and start of list
|
||||
/// items: `-`.
|
||||
/// The unary negation and binary subtraction operator: `-`.
|
||||
Minus,
|
||||
/// The division operator, start of term list items, and fraction operator
|
||||
/// in a formula: `/`.
|
||||
/// The division operator and fraction operator in math: `/`.
|
||||
Slash,
|
||||
/// The superscript operator in a formula: `^`.
|
||||
/// The superscript operator in math: `^`.
|
||||
Hat,
|
||||
/// The field access and method call operator: `.`.
|
||||
Dot,
|
||||
@ -119,59 +172,6 @@ pub enum SyntaxKind {
|
||||
/// The `as` keyword.
|
||||
As,
|
||||
|
||||
/// Markup of which all lines must have a minimal indentation.
|
||||
///
|
||||
/// Notably, the number does not determine in which column the markup
|
||||
/// started, but to the right of which column all markup elements must be,
|
||||
/// so it is zero except inside indent-aware constructs like lists.
|
||||
Markup { min_indent: usize },
|
||||
/// Plain text without markup.
|
||||
Text,
|
||||
/// A forced line break: `\`.
|
||||
Linebreak,
|
||||
/// An escape sequence: `\#`, `\u{1F5FA}`.
|
||||
Escape,
|
||||
/// A shorthand for a unicode codepoint. For example, `~` for non-breaking
|
||||
/// space or `-?` for a soft hyphen.
|
||||
Shorthand,
|
||||
/// Symbol notation: `:arrow:l:`. The string only contains the inner part
|
||||
/// without leading and trailing dot.
|
||||
Symbol,
|
||||
/// A smart quote: `'` or `"`.
|
||||
SmartQuote,
|
||||
/// Strong content: `*Strong*`.
|
||||
Strong,
|
||||
/// Emphasized content: `_Emphasized_`.
|
||||
Emph,
|
||||
/// Raw text with optional syntax highlighting: `` `...` ``.
|
||||
Raw { column: usize },
|
||||
/// A hyperlink: `https://typst.org`.
|
||||
Link,
|
||||
/// A label: `<intro>`.
|
||||
Label,
|
||||
/// A reference: `@target`.
|
||||
Ref,
|
||||
/// A section heading: `= Introduction`.
|
||||
Heading,
|
||||
/// An item in a bullet list: `- ...`.
|
||||
ListItem,
|
||||
/// An item in an enumeration (numbered list): `+ ...` or `1. ...`.
|
||||
EnumItem,
|
||||
/// An explicit enumeration numbering: `23.`.
|
||||
EnumNumbering,
|
||||
/// An item in a term list: `/ Term: Details`.
|
||||
TermItem,
|
||||
/// A mathematical formula: `$x$`, `$ x^2 $`.
|
||||
Math,
|
||||
/// An atom in a formula: `x`, `+`, `12`.
|
||||
Atom,
|
||||
/// A base with optional sub- and superscripts in a formula: `a_1^2`.
|
||||
Script,
|
||||
/// A fraction in a formula: `x/2`.
|
||||
Frac,
|
||||
/// An alignment point in a formula: `&`.
|
||||
AlignPoint,
|
||||
|
||||
/// An identifier: `it`.
|
||||
Ident,
|
||||
/// A boolean: `true`, `false`.
|
||||
@ -243,54 +243,103 @@ pub enum SyntaxKind {
|
||||
/// A return from a function: `return`, `return x + 1`.
|
||||
FuncReturn,
|
||||
|
||||
/// A line comment: `// ...`.
|
||||
LineComment,
|
||||
/// A block comment: `/* ... */`.
|
||||
BlockComment,
|
||||
/// An invalid sequence of characters.
|
||||
Error,
|
||||
/// The end of the file.
|
||||
Eof,
|
||||
}
|
||||
|
||||
impl SyntaxKind {
|
||||
/// Whether this is trivia.
|
||||
pub fn is_trivia(self) -> bool {
|
||||
self.is_space() || self.is_comment() || self.is_error()
|
||||
/// Is this a bracket, brace, or parenthesis?
|
||||
pub fn is_grouping(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::LeftBracket
|
||||
| Self::LeftBrace
|
||||
| Self::LeftParen
|
||||
| Self::RightBracket
|
||||
| Self::RightBrace
|
||||
| Self::RightParen
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this is a space.
|
||||
pub fn is_space(self) -> bool {
|
||||
matches!(self, Self::Space { .. })
|
||||
/// Does this node terminate a preceding expression?
|
||||
pub fn is_terminator(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Eof
|
||||
| Self::Semicolon
|
||||
| Self::RightBrace
|
||||
| Self::RightParen
|
||||
| Self::RightBracket
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this is a comment.
|
||||
pub fn is_comment(self) -> bool {
|
||||
matches!(self, Self::LineComment | Self::BlockComment)
|
||||
}
|
||||
|
||||
/// Whether this is an error.
|
||||
pub fn is_error(self) -> bool {
|
||||
matches!(self, SyntaxKind::Error)
|
||||
}
|
||||
|
||||
/// Whether this is a left or right parenthesis.
|
||||
pub fn is_paren(self) -> bool {
|
||||
matches!(self, Self::LeftParen | Self::RightParen)
|
||||
/// Is this a code or content block.
|
||||
pub fn is_block(self) -> bool {
|
||||
matches!(self, Self::CodeBlock | Self::ContentBlock)
|
||||
}
|
||||
|
||||
/// Does this node need termination through a semicolon or linebreak?
|
||||
pub fn is_stmt(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
SyntaxKind::LetBinding
|
||||
| SyntaxKind::SetRule
|
||||
| SyntaxKind::ShowRule
|
||||
| SyntaxKind::ModuleImport
|
||||
| SyntaxKind::ModuleInclude
|
||||
Self::LetBinding
|
||||
| Self::SetRule
|
||||
| Self::ShowRule
|
||||
| Self::ModuleImport
|
||||
| Self::ModuleInclude
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this kind of node is automatically skipped by the parser in
|
||||
/// code and math mode.
|
||||
pub fn is_trivia(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Space | Self::Parbreak | Self::LineComment | Self::BlockComment
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether this is an error.
|
||||
pub fn is_error(self) -> bool {
|
||||
self == Self::Error
|
||||
}
|
||||
|
||||
/// A human-readable name for the kind.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::LineComment => "line comment",
|
||||
Self::BlockComment => "block comment",
|
||||
Self::Space { .. } => "space",
|
||||
Self::Markup => "markup",
|
||||
Self::Text => "text",
|
||||
Self::Space => "space",
|
||||
Self::Linebreak => "line break",
|
||||
Self::Parbreak => "paragraph break",
|
||||
Self::Escape => "escape sequence",
|
||||
Self::Shorthand => "shorthand",
|
||||
Self::Symbol => "symbol notation",
|
||||
Self::Strong => "strong content",
|
||||
Self::Emph => "emphasized content",
|
||||
Self::Raw => "raw block",
|
||||
Self::Link => "link",
|
||||
Self::Label => "label",
|
||||
Self::Ref => "reference",
|
||||
Self::Heading => "heading",
|
||||
Self::HeadingMarker => "heading marker",
|
||||
Self::ListItem => "list item",
|
||||
Self::ListMarker => "list marker",
|
||||
Self::EnumItem => "enum item",
|
||||
Self::EnumMarker => "enum marker",
|
||||
Self::TermItem => "term list item",
|
||||
Self::TermMarker => "term marker",
|
||||
Self::Math => "math formula",
|
||||
Self::Atom => "math atom",
|
||||
Self::Script => "script",
|
||||
Self::Frac => "fraction",
|
||||
Self::AlignPoint => "alignment point",
|
||||
Self::LeftBrace => "opening brace",
|
||||
Self::RightBrace => "closing brace",
|
||||
Self::LeftBracket => "opening bracket",
|
||||
@ -309,7 +358,7 @@ impl SyntaxKind {
|
||||
Self::Slash => "slash",
|
||||
Self::Hat => "hat",
|
||||
Self::Dot => "dot",
|
||||
Self::Eq => "assignment operator",
|
||||
Self::Eq => "equals sign",
|
||||
Self::EqEq => "equality operator",
|
||||
Self::ExclEq => "inequality operator",
|
||||
Self::Lt => "less-than operator",
|
||||
@ -341,28 +390,6 @@ impl SyntaxKind {
|
||||
Self::Import => "keyword `import`",
|
||||
Self::Include => "keyword `include`",
|
||||
Self::As => "keyword `as`",
|
||||
Self::Markup { .. } => "markup",
|
||||
Self::Text => "text",
|
||||
Self::Linebreak => "linebreak",
|
||||
Self::Escape => "escape sequence",
|
||||
Self::Shorthand => "shorthand",
|
||||
Self::Symbol => "symbol notation",
|
||||
Self::Strong => "strong content",
|
||||
Self::Emph => "emphasized content",
|
||||
Self::Raw { .. } => "raw block",
|
||||
Self::Link => "link",
|
||||
Self::Label => "label",
|
||||
Self::Ref => "reference",
|
||||
Self::Heading => "heading",
|
||||
Self::ListItem => "list item",
|
||||
Self::EnumItem => "enumeration item",
|
||||
Self::EnumNumbering => "enumeration item numbering",
|
||||
Self::TermItem => "term list item",
|
||||
Self::Math => "math formula",
|
||||
Self::Atom => "math atom",
|
||||
Self::Script => "script",
|
||||
Self::Frac => "fraction",
|
||||
Self::AlignPoint => "alignment point",
|
||||
Self::Ident => "identifier",
|
||||
Self::Bool => "boolean",
|
||||
Self::Int => "integer",
|
||||
@ -398,7 +425,10 @@ impl SyntaxKind {
|
||||
Self::LoopBreak => "`break` expression",
|
||||
Self::LoopContinue => "`continue` expression",
|
||||
Self::FuncReturn => "`return` expression",
|
||||
Self::LineComment => "line comment",
|
||||
Self::BlockComment => "block comment",
|
||||
Self::Error => "syntax error",
|
||||
Self::Eof => "end of file",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,12 +9,11 @@ use crate::util::{format_eco, EcoString};
|
||||
pub(super) struct Lexer<'s> {
|
||||
/// The underlying scanner.
|
||||
s: Scanner<'s>,
|
||||
/// The mode the lexer is in. This determines what tokens it recognizes.
|
||||
/// The mode the lexer is in. This determines which kinds of tokens it
|
||||
/// produces.
|
||||
mode: LexMode,
|
||||
/// Whether the last token has been terminated.
|
||||
terminated: bool,
|
||||
/// Offsets the indentation on the first line of the source.
|
||||
column_offset: usize,
|
||||
/// Whether the last token contained a newline.
|
||||
newline: bool,
|
||||
/// An error for the last token.
|
||||
error: Option<(EcoString, ErrorPos)>,
|
||||
}
|
||||
@ -33,12 +32,11 @@ pub(super) enum LexMode {
|
||||
impl<'s> Lexer<'s> {
|
||||
/// Create a new lexer with the given mode and a prefix to offset column
|
||||
/// calculations.
|
||||
pub fn with_prefix(prefix: &str, text: &'s str, mode: LexMode) -> Self {
|
||||
pub fn new(text: &'s str, mode: LexMode) -> Self {
|
||||
Self {
|
||||
s: Scanner::new(text),
|
||||
mode,
|
||||
terminated: true,
|
||||
column_offset: column(prefix, prefix.len(), 0),
|
||||
newline: false,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
@ -64,26 +62,18 @@ impl<'s> Lexer<'s> {
|
||||
self.s.jump(index);
|
||||
}
|
||||
|
||||
/// The underlying scanner.
|
||||
pub fn scanner(&self) -> Scanner<'s> {
|
||||
self.s
|
||||
/// Whether the last token contained a newline.
|
||||
pub fn newline(&self) -> bool {
|
||||
self.newline
|
||||
}
|
||||
|
||||
/// Whether the last token was terminated.
|
||||
pub fn terminated(&self) -> bool {
|
||||
self.terminated
|
||||
}
|
||||
|
||||
/// The column index of a given index in the source string.
|
||||
pub fn column(&self, index: usize) -> usize {
|
||||
column(self.s.string(), index, self.column_offset)
|
||||
}
|
||||
|
||||
/// Take out the last error.
|
||||
pub fn last_error(&mut self) -> Option<(EcoString, ErrorPos)> {
|
||||
/// Take out the last error, if any.
|
||||
pub fn take_error(&mut self) -> Option<(EcoString, ErrorPos)> {
|
||||
self.error.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Lexer<'_> {
|
||||
/// Construct a full-positioned syntax error.
|
||||
fn error(&mut self, message: impl Into<EcoString>) -> SyntaxKind {
|
||||
self.error = Some((message.into(), ErrorPos::Full));
|
||||
@ -97,45 +87,53 @@ impl<'s> Lexer<'s> {
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for Lexer<'_> {
|
||||
type Item = SyntaxKind;
|
||||
|
||||
/// Produce the next token.
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
/// Shared.
|
||||
impl Lexer<'_> {
|
||||
pub fn next(&mut self) -> SyntaxKind {
|
||||
self.newline = false;
|
||||
self.error = None;
|
||||
let start = self.s.cursor();
|
||||
let c = self.s.eat()?;
|
||||
Some(match c {
|
||||
// Trivia.
|
||||
c if c.is_whitespace() => self.whitespace(c),
|
||||
'/' if self.s.eat_if('/') => self.line_comment(),
|
||||
'/' if self.s.eat_if('*') => self.block_comment(),
|
||||
'*' if self.s.eat_if('/') => self.error("unexpected end of block comment"),
|
||||
match self.s.eat() {
|
||||
Some(c) if c.is_whitespace() => self.whitespace(start, c),
|
||||
Some('/') if self.s.eat_if('/') => self.line_comment(),
|
||||
Some('/') if self.s.eat_if('*') => self.block_comment(),
|
||||
Some('*') if self.s.eat_if('/') => {
|
||||
self.error("unexpected end of block comment")
|
||||
}
|
||||
|
||||
// Other things.
|
||||
_ => match self.mode {
|
||||
Some(c) => match self.mode {
|
||||
LexMode::Markup => self.markup(start, c),
|
||||
LexMode::Math => self.math(c),
|
||||
LexMode::Code => self.code(start, c),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared.
|
||||
impl Lexer<'_> {
|
||||
None => SyntaxKind::Eof,
|
||||
}
|
||||
}
|
||||
|
||||
fn whitespace(&mut self, start: usize, c: char) -> SyntaxKind {
|
||||
let more = self.s.eat_while(char::is_whitespace);
|
||||
let newlines = match c {
|
||||
' ' if more.is_empty() => 0,
|
||||
_ => count_newlines(self.s.from(start)),
|
||||
};
|
||||
|
||||
self.newline = newlines > 0;
|
||||
if self.mode == LexMode::Markup && newlines >= 2 {
|
||||
SyntaxKind::Parbreak
|
||||
} else {
|
||||
SyntaxKind::Space
|
||||
}
|
||||
}
|
||||
|
||||
fn line_comment(&mut self) -> SyntaxKind {
|
||||
self.s.eat_until(is_newline);
|
||||
if self.s.done() {
|
||||
self.terminated = false;
|
||||
}
|
||||
SyntaxKind::LineComment
|
||||
}
|
||||
|
||||
fn block_comment(&mut self) -> SyntaxKind {
|
||||
let mut state = '_';
|
||||
let mut depth = 1;
|
||||
self.terminated = false;
|
||||
|
||||
// Find the first `*/` that does not correspond to a nested `/*`.
|
||||
while let Some(c) = self.s.eat() {
|
||||
@ -143,7 +141,6 @@ impl Lexer<'_> {
|
||||
('*', '/') => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
self.terminated = true;
|
||||
break;
|
||||
}
|
||||
'_'
|
||||
@ -162,32 +159,6 @@ impl Lexer<'_> {
|
||||
|
||||
SyntaxKind::BlockComment
|
||||
}
|
||||
|
||||
fn whitespace(&mut self, c: char) -> SyntaxKind {
|
||||
if c == ' ' && !self.s.at(char::is_whitespace) {
|
||||
return SyntaxKind::Space { newlines: 0 };
|
||||
}
|
||||
|
||||
self.s.uneat();
|
||||
|
||||
// Count the number of newlines.
|
||||
let mut newlines = 0;
|
||||
while let Some(c) = self.s.eat() {
|
||||
if !c.is_whitespace() {
|
||||
self.s.uneat();
|
||||
break;
|
||||
}
|
||||
|
||||
if is_newline(c) {
|
||||
if c == '\r' {
|
||||
self.s.eat_if('\n');
|
||||
}
|
||||
newlines += 1;
|
||||
}
|
||||
}
|
||||
|
||||
SyntaxKind::Space { newlines }
|
||||
}
|
||||
}
|
||||
|
||||
/// Markup.
|
||||
@ -199,9 +170,9 @@ impl Lexer<'_> {
|
||||
'`' => self.raw(),
|
||||
'h' if self.s.eat_if("ttp://") => self.link(),
|
||||
'h' if self.s.eat_if("ttps://") => self.link(),
|
||||
'0'..='9' => self.numbering(start),
|
||||
'<' if self.s.at(is_id_continue) => self.label(),
|
||||
'@' if self.s.at(is_id_continue) => self.reference(),
|
||||
'0'..='9' => self.numbering(start),
|
||||
'#' if self.s.eat_if('{') => SyntaxKind::LeftBrace,
|
||||
'#' if self.s.eat_if('[') => SyntaxKind::LeftBracket,
|
||||
'#' if self.s.at(is_id_start) => {
|
||||
@ -225,17 +196,154 @@ impl Lexer<'_> {
|
||||
'\'' => SyntaxKind::SmartQuote,
|
||||
'"' => SyntaxKind::SmartQuote,
|
||||
'$' => SyntaxKind::Dollar,
|
||||
'=' => SyntaxKind::Eq,
|
||||
'+' => SyntaxKind::Plus,
|
||||
'/' => SyntaxKind::Slash,
|
||||
'~' => SyntaxKind::Shorthand,
|
||||
':' => SyntaxKind::Colon,
|
||||
'-' => SyntaxKind::Minus,
|
||||
'=' => {
|
||||
self.s.eat_while('=');
|
||||
if self.space_and_more() {
|
||||
SyntaxKind::HeadingMarker
|
||||
} else {
|
||||
self.text()
|
||||
}
|
||||
}
|
||||
'-' if self.space_and_more() => SyntaxKind::ListMarker,
|
||||
'+' if self.space_and_more() => SyntaxKind::EnumMarker,
|
||||
'/' if self.space_and_more() => SyntaxKind::TermMarker,
|
||||
|
||||
_ => self.text(),
|
||||
}
|
||||
}
|
||||
|
||||
fn backslash(&mut self) -> SyntaxKind {
|
||||
if self.s.eat_if("u{") {
|
||||
let hex = self.s.eat_while(char::is_ascii_alphanumeric);
|
||||
if !self.s.eat_if('}') {
|
||||
return self.error_at_end("expected closing brace");
|
||||
}
|
||||
|
||||
if u32::from_str_radix(hex, 16)
|
||||
.ok()
|
||||
.and_then(std::char::from_u32)
|
||||
.is_none()
|
||||
{
|
||||
return self.error("invalid unicode escape sequence");
|
||||
}
|
||||
|
||||
return SyntaxKind::Escape;
|
||||
}
|
||||
|
||||
if self.s.done() || self.s.at(char::is_whitespace) {
|
||||
SyntaxKind::Linebreak
|
||||
} else {
|
||||
self.s.eat();
|
||||
SyntaxKind::Escape
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_symbol(&mut self) -> SyntaxKind {
|
||||
let start = self.s.cursor();
|
||||
let mut end = start;
|
||||
while !self.s.eat_while(is_id_continue).is_empty() && self.s.at(':') {
|
||||
end = self.s.cursor();
|
||||
self.s.eat();
|
||||
}
|
||||
|
||||
self.s.jump(end);
|
||||
|
||||
if start < end {
|
||||
self.s.expect(':');
|
||||
SyntaxKind::Symbol
|
||||
} else if self.mode == LexMode::Markup {
|
||||
SyntaxKind::Colon
|
||||
} else {
|
||||
SyntaxKind::Atom
|
||||
}
|
||||
}
|
||||
|
||||
fn raw(&mut self) -> SyntaxKind {
|
||||
let mut backticks = 1;
|
||||
while self.s.eat_if('`') {
|
||||
backticks += 1;
|
||||
}
|
||||
|
||||
if backticks == 2 {
|
||||
return SyntaxKind::Raw;
|
||||
}
|
||||
|
||||
let mut found = 0;
|
||||
while found < backticks {
|
||||
match self.s.eat() {
|
||||
Some('`') => found += 1,
|
||||
Some(_) => found = 0,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
if found != backticks {
|
||||
let remaining = backticks - found;
|
||||
let noun = if remaining == 1 { "backtick" } else { "backticks" };
|
||||
return self.error_at_end(if found == 0 {
|
||||
format_eco!("expected {} {}", remaining, noun)
|
||||
} else {
|
||||
format_eco!("expected {} more {}", remaining, noun)
|
||||
});
|
||||
}
|
||||
|
||||
SyntaxKind::Raw
|
||||
}
|
||||
|
||||
fn link(&mut self) -> SyntaxKind {
|
||||
#[rustfmt::skip]
|
||||
self.s.eat_while(|c: char| matches!(c,
|
||||
| '0' ..= '9'
|
||||
| 'a' ..= 'z'
|
||||
| 'A' ..= 'Z'
|
||||
| '~' | '/' | '%' | '?' | '#' | '&' | '+' | '='
|
||||
| '\'' | '.' | ',' | ';'
|
||||
));
|
||||
|
||||
if self.s.scout(-1) == Some('.') {
|
||||
self.s.uneat();
|
||||
}
|
||||
|
||||
SyntaxKind::Link
|
||||
}
|
||||
|
||||
fn numbering(&mut self, start: usize) -> SyntaxKind {
|
||||
self.s.eat_while(char::is_ascii_digit);
|
||||
|
||||
let read = self.s.from(start);
|
||||
if self.s.eat_if('.') {
|
||||
if let Ok(number) = read.parse::<usize>() {
|
||||
if number == 0 {
|
||||
return self.error("must be positive");
|
||||
}
|
||||
|
||||
return SyntaxKind::EnumMarker;
|
||||
}
|
||||
}
|
||||
|
||||
self.text()
|
||||
}
|
||||
|
||||
fn label(&mut self) -> SyntaxKind {
|
||||
let label = self.s.eat_while(is_id_continue);
|
||||
if label.is_empty() {
|
||||
return self.error("label cannot be empty");
|
||||
}
|
||||
|
||||
if !self.s.eat_if('>') {
|
||||
return self.error_at_end("expected closing angle bracket");
|
||||
}
|
||||
|
||||
SyntaxKind::Label
|
||||
}
|
||||
|
||||
fn reference(&mut self) -> SyntaxKind {
|
||||
self.s.eat_while(is_id_continue);
|
||||
SyntaxKind::Ref
|
||||
}
|
||||
|
||||
fn text(&mut self) -> SyntaxKind {
|
||||
macro_rules! table {
|
||||
($(|$c:literal)*) => {
|
||||
@ -277,132 +385,20 @@ impl Lexer<'_> {
|
||||
SyntaxKind::Text
|
||||
}
|
||||
|
||||
fn backslash(&mut self) -> SyntaxKind {
|
||||
if self.s.eat_if("u{") {
|
||||
let hex = self.s.eat_while(char::is_ascii_alphanumeric);
|
||||
if !self.s.eat_if('}') {
|
||||
self.terminated = false;
|
||||
return self.error_at_end("expected closing brace");
|
||||
}
|
||||
|
||||
if u32::from_str_radix(hex, 16)
|
||||
.ok()
|
||||
.and_then(std::char::from_u32)
|
||||
.is_none()
|
||||
{
|
||||
return self.error("invalid unicode escape sequence");
|
||||
}
|
||||
|
||||
return SyntaxKind::Escape;
|
||||
}
|
||||
|
||||
if self.s.done() || self.s.at(char::is_whitespace) {
|
||||
SyntaxKind::Linebreak
|
||||
} else {
|
||||
self.s.eat();
|
||||
SyntaxKind::Escape
|
||||
}
|
||||
}
|
||||
|
||||
fn maybe_symbol(&mut self) -> SyntaxKind {
|
||||
let start = self.s.cursor();
|
||||
let mut end = start;
|
||||
while !self.s.eat_while(is_id_continue).is_empty() && self.s.at(':') {
|
||||
end = self.s.cursor();
|
||||
self.s.eat();
|
||||
}
|
||||
|
||||
self.s.jump(end);
|
||||
|
||||
if start < end {
|
||||
self.s.expect(':');
|
||||
SyntaxKind::Symbol
|
||||
} else if self.mode == LexMode::Markup {
|
||||
SyntaxKind::Colon
|
||||
} else {
|
||||
SyntaxKind::Atom
|
||||
}
|
||||
}
|
||||
|
||||
fn link(&mut self) -> SyntaxKind {
|
||||
#[rustfmt::skip]
|
||||
self.s.eat_while(|c: char| matches!(c,
|
||||
| '0' ..= '9'
|
||||
| 'a' ..= 'z'
|
||||
| 'A' ..= 'Z'
|
||||
| '~' | '/' | '%' | '?' | '#' | '&' | '+' | '='
|
||||
| '\'' | '.' | ',' | ';'
|
||||
));
|
||||
|
||||
if self.s.scout(-1) == Some('.') {
|
||||
self.s.uneat();
|
||||
}
|
||||
|
||||
SyntaxKind::Link
|
||||
}
|
||||
|
||||
fn raw(&mut self) -> SyntaxKind {
|
||||
let column = self.column(self.s.cursor() - 1);
|
||||
|
||||
let mut backticks = 1;
|
||||
while self.s.eat_if('`') {
|
||||
backticks += 1;
|
||||
}
|
||||
|
||||
if backticks == 2 {
|
||||
return SyntaxKind::Raw { column };
|
||||
}
|
||||
|
||||
let mut found = 0;
|
||||
while found < backticks {
|
||||
match self.s.eat() {
|
||||
Some('`') => found += 1,
|
||||
Some(_) => found = 0,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
|
||||
if found != backticks {
|
||||
self.terminated = false;
|
||||
let remaining = backticks - found;
|
||||
let noun = if remaining == 1 { "backtick" } else { "backticks" };
|
||||
return self.error_at_end(if found == 0 {
|
||||
format_eco!("expected {} {}", remaining, noun)
|
||||
} else {
|
||||
format_eco!("expected {} more {}", remaining, noun)
|
||||
});
|
||||
}
|
||||
|
||||
SyntaxKind::Raw { column }
|
||||
}
|
||||
|
||||
fn numbering(&mut self, start: usize) -> SyntaxKind {
|
||||
self.s.eat_while(char::is_ascii_digit);
|
||||
|
||||
let read = self.s.from(start);
|
||||
if self.s.eat_if('.') {
|
||||
if let Ok(number) = read.parse::<usize>() {
|
||||
if number == 0 {
|
||||
return self.error("must be positive");
|
||||
}
|
||||
|
||||
return SyntaxKind::EnumNumbering;
|
||||
}
|
||||
}
|
||||
|
||||
self.text()
|
||||
}
|
||||
|
||||
fn reference(&mut self) -> SyntaxKind {
|
||||
self.s.eat_while(is_id_continue);
|
||||
SyntaxKind::Ref
|
||||
}
|
||||
|
||||
fn in_word(&self) -> bool {
|
||||
let alphanumeric = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
|
||||
let alphanum = |c: Option<char>| c.map_or(false, |c| c.is_alphanumeric());
|
||||
let prev = self.s.scout(-2);
|
||||
let next = self.s.peek();
|
||||
alphanumeric(prev) && alphanumeric(next)
|
||||
alphanum(prev) && alphanum(next)
|
||||
}
|
||||
|
||||
fn space_and_more(&self) -> bool {
|
||||
let mut s = self.s;
|
||||
if !s.at(char::is_whitespace) {
|
||||
return false;
|
||||
}
|
||||
s.eat_while(|c: char| c.is_whitespace() && !is_newline(c));
|
||||
!s.done() && !s.at(is_newline)
|
||||
}
|
||||
}
|
||||
|
||||
@ -586,26 +582,11 @@ impl Lexer<'_> {
|
||||
});
|
||||
|
||||
if !self.s.eat_if('"') {
|
||||
self.terminated = false;
|
||||
return self.error_at_end("expected quote");
|
||||
}
|
||||
|
||||
SyntaxKind::Str
|
||||
}
|
||||
|
||||
fn label(&mut self) -> SyntaxKind {
|
||||
let label = self.s.eat_while(is_id_continue);
|
||||
if label.is_empty() {
|
||||
return self.error("label cannot be empty");
|
||||
}
|
||||
|
||||
if !self.s.eat_if('>') {
|
||||
self.terminated = false;
|
||||
return self.error_at_end("expected closing angle bracket");
|
||||
}
|
||||
|
||||
SyntaxKind::Label
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to parse an identifier into a keyword.
|
||||
@ -632,34 +613,6 @@ fn keyword(ident: &str) -> Option<SyntaxKind> {
|
||||
})
|
||||
}
|
||||
|
||||
/// The column index of a given index in the source string, given a column
|
||||
/// offset for the first line.
|
||||
fn column(string: &str, index: usize, offset: usize) -> usize {
|
||||
let mut apply_offset = false;
|
||||
let res = string[..index]
|
||||
.char_indices()
|
||||
.rev()
|
||||
.take_while(|&(_, c)| !is_newline(c))
|
||||
.inspect(|&(i, _)| {
|
||||
if i == 0 {
|
||||
apply_offset = true
|
||||
}
|
||||
})
|
||||
.count();
|
||||
|
||||
// The loop is never executed if the slice is empty, but we are of
|
||||
// course still at the start of the first line.
|
||||
if index == 0 {
|
||||
apply_offset = true;
|
||||
}
|
||||
|
||||
if apply_offset {
|
||||
res + offset
|
||||
} else {
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this character denotes a newline.
|
||||
#[inline]
|
||||
pub fn is_newline(character: char) -> bool {
|
||||
@ -695,6 +648,21 @@ pub(super) fn split_newlines(text: &str) -> Vec<&str> {
|
||||
lines
|
||||
}
|
||||
|
||||
/// Count the number of newlines in text.
|
||||
fn count_newlines(text: &str) -> usize {
|
||||
let mut newlines = 0;
|
||||
let mut s = Scanner::new(text);
|
||||
while let Some(c) = s.eat() {
|
||||
if is_newline(c) {
|
||||
if c == '\r' {
|
||||
s.eat_if('\n');
|
||||
}
|
||||
newlines += 1;
|
||||
}
|
||||
}
|
||||
newlines
|
||||
}
|
||||
|
||||
/// Whether a string is a valid unicode identifier.
|
||||
///
|
||||
/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
|
||||
|
||||
@ -6,7 +6,7 @@ mod kind;
|
||||
mod lexer;
|
||||
mod node;
|
||||
mod parser;
|
||||
mod reparse;
|
||||
mod reparser;
|
||||
mod source;
|
||||
mod span;
|
||||
|
||||
@ -14,5 +14,6 @@ pub use self::kind::*;
|
||||
pub use self::lexer::*;
|
||||
pub use self::node::*;
|
||||
pub use self::parser::*;
|
||||
pub use self::reparser::*;
|
||||
pub use self::source::*;
|
||||
pub use self::span::*;
|
||||
|
||||
@ -12,15 +12,15 @@ use crate::util::EcoString;
|
||||
#[derive(Clone, PartialEq, Hash)]
|
||||
pub struct SyntaxNode(Repr);
|
||||
|
||||
/// The two internal representations.
|
||||
/// The three internal representations.
|
||||
#[derive(Clone, PartialEq, Hash)]
|
||||
enum Repr {
|
||||
/// A leaf node.
|
||||
Leaf(LeafNode),
|
||||
/// A reference-counted inner node.
|
||||
Inner(Arc<InnerNode>),
|
||||
/// An error.
|
||||
Error(ErrorNode),
|
||||
/// An error node.
|
||||
Error(Arc<ErrorNode>),
|
||||
}
|
||||
|
||||
impl SyntaxNode {
|
||||
@ -36,7 +36,7 @@ impl SyntaxNode {
|
||||
|
||||
/// Create a new error node.
|
||||
pub fn error(message: impl Into<EcoString>, pos: ErrorPos, len: usize) -> Self {
|
||||
Self(Repr::Error(ErrorNode::new(message, pos, len)))
|
||||
Self(Repr::Error(Arc::new(ErrorNode::new(message, pos, len))))
|
||||
}
|
||||
|
||||
/// The type of the node.
|
||||
@ -134,17 +134,13 @@ impl SyntaxNode {
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Change the type of the node.
|
||||
pub(super) fn convert_to(&mut self, kind: SyntaxKind) {
|
||||
debug_assert!(!kind.is_error());
|
||||
match &mut self.0 {
|
||||
Repr::Leaf(leaf) => leaf.kind = kind,
|
||||
Repr::Inner(inner) => {
|
||||
let node = Arc::make_mut(inner);
|
||||
node.kind = kind;
|
||||
}
|
||||
Repr::Error(_) => {}
|
||||
impl SyntaxNode {
|
||||
/// Mark this node as erroneous.
|
||||
pub(super) fn make_erroneous(&mut self) {
|
||||
if let Repr::Inner(inner) = &mut self.0 {
|
||||
Arc::make_mut(inner).erroneous = true;
|
||||
}
|
||||
}
|
||||
|
||||
@ -159,7 +155,7 @@ impl SyntaxNode {
|
||||
match &mut self.0 {
|
||||
Repr::Leaf(leaf) => leaf.span = span,
|
||||
Repr::Inner(inner) => Arc::make_mut(inner).synthesize(span),
|
||||
Repr::Error(error) => error.span = span,
|
||||
Repr::Error(error) => Arc::make_mut(error).span = span,
|
||||
}
|
||||
}
|
||||
|
||||
@ -177,7 +173,7 @@ impl SyntaxNode {
|
||||
match &mut self.0 {
|
||||
Repr::Leaf(leaf) => leaf.span = mid,
|
||||
Repr::Inner(inner) => Arc::make_mut(inner).numberize(id, None, within)?,
|
||||
Repr::Error(error) => error.span = mid,
|
||||
Repr::Error(error) => Arc::make_mut(error).span = mid,
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@ -245,7 +241,7 @@ impl SyntaxNode {
|
||||
}
|
||||
|
||||
/// The upper bound of assigned numbers in this subtree.
|
||||
fn upper(&self) -> u64 {
|
||||
pub(super) fn upper(&self) -> u64 {
|
||||
match &self.0 {
|
||||
Repr::Inner(inner) => inner.upper,
|
||||
Repr::Leaf(leaf) => leaf.span.number() + 1,
|
||||
@ -297,7 +293,7 @@ impl LeafNode {
|
||||
|
||||
impl Debug for LeafNode {
|
||||
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
|
||||
write!(f, "{:?}: {}", self.kind, self.len())
|
||||
write!(f, "{:?}: {:?}", self.kind, self.text)
|
||||
}
|
||||
}
|
||||
|
||||
@ -588,7 +584,7 @@ impl ErrorNode {
|
||||
|
||||
impl Debug for ErrorNode {
|
||||
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
|
||||
write!(f, "({}): {}", self.message, self.len)
|
||||
write!(f, "Error: {} ({})", self.len, self.message)
|
||||
}
|
||||
}
|
||||
|
||||
@ -888,7 +884,7 @@ mod tests {
|
||||
let prev = leaf.prev_leaf().unwrap();
|
||||
let next = leaf.next_leaf().unwrap();
|
||||
assert_eq!(prev.kind(), SyntaxKind::Eq);
|
||||
assert_eq!(leaf.kind(), SyntaxKind::Space { newlines: 0 });
|
||||
assert_eq!(leaf.kind(), SyntaxKind::Space);
|
||||
assert_eq!(next.kind(), SyntaxKind::Int);
|
||||
}
|
||||
}
|
||||
|
||||
2348
src/syntax/parser.rs
2348
src/syntax/parser.rs
File diff suppressed because it is too large
Load Diff
@ -1,525 +0,0 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use super::{
|
||||
is_newline, parse, reparse_code_block, reparse_content_block,
|
||||
reparse_markup_elements, Span, SyntaxKind, SyntaxNode,
|
||||
};
|
||||
|
||||
/// Refresh the given syntax node with as little parsing as possible.
|
||||
///
|
||||
/// Takes the new source, the range in the old source that was replaced and the
|
||||
/// length of the replacement.
|
||||
///
|
||||
/// Returns the range in the new source that was ultimately reparsed.
|
||||
pub fn reparse(
|
||||
root: &mut SyntaxNode,
|
||||
text: &str,
|
||||
replaced: Range<usize>,
|
||||
replacement_len: usize,
|
||||
) -> Range<usize> {
|
||||
let change = Change { text, replaced, replacement_len };
|
||||
if let Some(range) = try_reparse(&change, root, 0, true, true) {
|
||||
return range;
|
||||
}
|
||||
|
||||
let id = root.span().source();
|
||||
*root = parse(text);
|
||||
root.numberize(id, Span::FULL).unwrap();
|
||||
0..text.len()
|
||||
}
|
||||
|
||||
/// Try to reparse inside the given node.
|
||||
fn try_reparse(
|
||||
change: &Change,
|
||||
node: &mut SyntaxNode,
|
||||
mut offset: usize,
|
||||
outermost: bool,
|
||||
safe_to_replace: bool,
|
||||
) -> Option<Range<usize>> {
|
||||
let is_markup = matches!(node.kind(), SyntaxKind::Markup { .. });
|
||||
let original_count = node.children().len();
|
||||
let original_offset = offset;
|
||||
|
||||
let mut search = SearchState::default();
|
||||
let mut ahead: Option<Ahead> = None;
|
||||
|
||||
// Whether the first node that should be replaced is at start.
|
||||
let mut at_start = true;
|
||||
|
||||
// Whether the last searched child is the outermost child.
|
||||
let mut child_outermost = false;
|
||||
|
||||
// Find the the first child in the range of children to reparse.
|
||||
for (i, child) in node.children().enumerate() {
|
||||
let pos = NodePos { idx: i, offset };
|
||||
let child_span = offset..offset + child.len();
|
||||
child_outermost = outermost && i + 1 == original_count;
|
||||
|
||||
match search {
|
||||
SearchState::NoneFound => {
|
||||
// The edit is contained within the span of the current element.
|
||||
if child_span.contains(&change.replaced.start)
|
||||
&& child_span.end >= change.replaced.end
|
||||
{
|
||||
// In Markup mode, we want to consider a non-whitespace
|
||||
// neighbor if the edit is on the node boundary.
|
||||
search = if is_markup && child_span.end == change.replaced.end {
|
||||
SearchState::RequireNonTrivia(pos)
|
||||
} else {
|
||||
SearchState::Contained(pos)
|
||||
};
|
||||
} else if child_span.contains(&change.replaced.start) {
|
||||
search = SearchState::Inside(pos);
|
||||
} else if child_span.end == change.replaced.start
|
||||
&& change.replaced.start == change.replaced.end
|
||||
&& child_outermost
|
||||
{
|
||||
search = SearchState::SpanFound(pos, pos);
|
||||
} else {
|
||||
// Update compulsary state of `ahead_nontrivia`.
|
||||
if let Some(ahead_nontrivia) = ahead.as_mut() {
|
||||
if let SyntaxKind::Space { newlines: (1..) } = child.kind() {
|
||||
ahead_nontrivia.newline();
|
||||
}
|
||||
}
|
||||
|
||||
// We look only for non spaces, non-semicolon and also
|
||||
// reject text that points to the special case for URL
|
||||
// evasion and line comments.
|
||||
if !child.kind().is_space()
|
||||
&& child.kind() != SyntaxKind::Semicolon
|
||||
&& (child.kind() != SyntaxKind::Text || child.text() != "/")
|
||||
&& (ahead.is_none() || change.replaced.start > child_span.end)
|
||||
&& !ahead.map_or(false, Ahead::is_compulsory)
|
||||
{
|
||||
ahead = Some(Ahead::new(pos, at_start, is_bounded(child.kind())));
|
||||
}
|
||||
|
||||
at_start = next_at_start(child.kind(), at_start);
|
||||
}
|
||||
}
|
||||
SearchState::Inside(start) => {
|
||||
if child_span.end == change.replaced.end {
|
||||
search = SearchState::RequireNonTrivia(start);
|
||||
} else if child_span.end > change.replaced.end {
|
||||
search = SearchState::SpanFound(start, pos);
|
||||
}
|
||||
}
|
||||
SearchState::RequireNonTrivia(start) => {
|
||||
if !child.kind().is_trivia() {
|
||||
search = SearchState::SpanFound(start, pos);
|
||||
}
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
|
||||
offset += child.len();
|
||||
|
||||
if search.done().is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If we were looking for a non-whitespace element and hit the end of
|
||||
// the file here, we instead use EOF as the end of the span.
|
||||
if let SearchState::RequireNonTrivia(start) = search {
|
||||
search = SearchState::SpanFound(
|
||||
start,
|
||||
NodePos {
|
||||
idx: node.children().len() - 1,
|
||||
offset: offset - node.children().last().unwrap().len(),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
if let SearchState::Contained(pos) = search {
|
||||
// Do not allow replacement of elements inside of constructs whose
|
||||
// opening and closing brackets look the same.
|
||||
let safe_inside = is_bounded(node.kind());
|
||||
let child = &mut node.children_mut()[pos.idx];
|
||||
let prev_len = child.len();
|
||||
let prev_descendants = child.descendants();
|
||||
|
||||
if !child.is_leaf() {
|
||||
if let Some(range) =
|
||||
try_reparse(change, child, pos.offset, child_outermost, safe_inside)
|
||||
{
|
||||
let new_len = child.len();
|
||||
let new_descendants = child.descendants();
|
||||
node.update_parent(prev_len, new_len, prev_descendants, new_descendants);
|
||||
return Some(range);
|
||||
}
|
||||
}
|
||||
|
||||
let superseded_span = pos.offset..pos.offset + prev_len;
|
||||
let func: Option<ReparseMode> = match child.kind() {
|
||||
SyntaxKind::CodeBlock => Some(ReparseMode::Code),
|
||||
SyntaxKind::ContentBlock => Some(ReparseMode::Content),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
// Return if the element was reparsable on its own, otherwise try to
|
||||
// treat it as a markup element.
|
||||
if let Some(func) = func {
|
||||
if let Some(result) = replace(
|
||||
change,
|
||||
node,
|
||||
func,
|
||||
pos.idx..pos.idx + 1,
|
||||
superseded_span,
|
||||
outermost,
|
||||
) {
|
||||
return Some(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure this is a markup node and that we may replace. If so, save
|
||||
// the current indent.
|
||||
let min_indent = match node.kind() {
|
||||
SyntaxKind::Markup { min_indent } if safe_to_replace => min_indent,
|
||||
_ => return None,
|
||||
};
|
||||
|
||||
let (mut start, end) = search.done()?;
|
||||
if let Some(ahead) = ahead {
|
||||
if start.offset == change.replaced.start || ahead.is_compulsory() {
|
||||
start = ahead.pos;
|
||||
at_start = ahead.at_start;
|
||||
}
|
||||
} else {
|
||||
start = NodePos { idx: 0, offset: original_offset };
|
||||
}
|
||||
|
||||
let superseded_span =
|
||||
start.offset..end.offset + node.children().as_slice()[end.idx].len();
|
||||
|
||||
replace(
|
||||
change,
|
||||
node,
|
||||
ReparseMode::MarkupElements { at_start, min_indent },
|
||||
start.idx..end.idx + 1,
|
||||
superseded_span,
|
||||
outermost,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reparse the superseded nodes and replace them.
|
||||
fn replace(
|
||||
change: &Change,
|
||||
node: &mut SyntaxNode,
|
||||
mode: ReparseMode,
|
||||
superseded_idx: Range<usize>,
|
||||
superseded_span: Range<usize>,
|
||||
outermost: bool,
|
||||
) -> Option<Range<usize>> {
|
||||
let superseded_start = superseded_idx.start;
|
||||
|
||||
let differential: isize =
|
||||
change.replacement_len as isize - change.replaced.len() as isize;
|
||||
let newborn_end = (superseded_span.end as isize + differential) as usize;
|
||||
let newborn_span = superseded_span.start..newborn_end;
|
||||
|
||||
let mut prefix = "";
|
||||
for (i, c) in change.text[..newborn_span.start].char_indices().rev() {
|
||||
if is_newline(c) {
|
||||
break;
|
||||
}
|
||||
prefix = &change.text[i..newborn_span.start];
|
||||
}
|
||||
|
||||
let (newborns, terminated, amount) = match mode {
|
||||
ReparseMode::Code => reparse_code_block(
|
||||
prefix,
|
||||
&change.text[newborn_span.start..],
|
||||
newborn_span.len(),
|
||||
),
|
||||
ReparseMode::Content => reparse_content_block(
|
||||
prefix,
|
||||
&change.text[newborn_span.start..],
|
||||
newborn_span.len(),
|
||||
),
|
||||
ReparseMode::MarkupElements { at_start, min_indent } => reparse_markup_elements(
|
||||
prefix,
|
||||
&change.text[newborn_span.start..],
|
||||
newborn_span.len(),
|
||||
differential,
|
||||
&node.children().as_slice()[superseded_start..],
|
||||
at_start,
|
||||
min_indent,
|
||||
),
|
||||
}?;
|
||||
|
||||
// Do not accept unclosed nodes if the old node wasn't at the right edge
|
||||
// of the tree.
|
||||
if !outermost && !terminated {
|
||||
return None;
|
||||
}
|
||||
|
||||
node.replace_children(superseded_start..superseded_start + amount, newborns)
|
||||
.ok()?;
|
||||
|
||||
Some(newborn_span)
|
||||
}
|
||||
|
||||
/// A description of a change.
|
||||
struct Change<'a> {
|
||||
/// The new source code, with the change applied.
|
||||
text: &'a str,
|
||||
/// Which range in the old source file was changed.
|
||||
replaced: Range<usize>,
|
||||
/// How many characters replaced the text in `replaced`.
|
||||
replacement_len: usize,
|
||||
}
|
||||
|
||||
/// Encodes the state machine of the search for the nodes are pending for
|
||||
/// replacement.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum SearchState {
|
||||
/// Neither an end nor a start have been found as of now.
|
||||
/// The latest non-trivia child is continually saved.
|
||||
NoneFound,
|
||||
/// The search has concluded by finding a node that fully contains the
|
||||
/// modifications.
|
||||
Contained(NodePos),
|
||||
/// The search has found the start of the modified nodes.
|
||||
Inside(NodePos),
|
||||
/// The search has found the end of the modified nodes but the change
|
||||
/// touched its boundries so another non-trivia node is needed.
|
||||
RequireNonTrivia(NodePos),
|
||||
/// The search has concluded by finding a start and an end index for nodes
|
||||
/// with a pending reparse.
|
||||
SpanFound(NodePos, NodePos),
|
||||
}
|
||||
|
||||
impl Default for SearchState {
|
||||
fn default() -> Self {
|
||||
Self::NoneFound
|
||||
}
|
||||
}
|
||||
|
||||
impl SearchState {
|
||||
fn done(self) -> Option<(NodePos, NodePos)> {
|
||||
match self {
|
||||
Self::NoneFound => None,
|
||||
Self::Contained(s) => Some((s, s)),
|
||||
Self::Inside(_) => None,
|
||||
Self::RequireNonTrivia(_) => None,
|
||||
Self::SpanFound(s, e) => Some((s, e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The position of a syntax node.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
struct NodePos {
|
||||
/// The index in the parent node.
|
||||
idx: usize,
|
||||
/// The byte offset in the string.
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
/// An ahead node with an index and whether it is `at_start`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
struct Ahead {
|
||||
/// The position of the node.
|
||||
pos: NodePos,
|
||||
/// The `at_start` before this node.
|
||||
at_start: bool,
|
||||
/// The kind of ahead node.
|
||||
kind: AheadKind,
|
||||
}
|
||||
|
||||
/// The kind of ahead node.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum AheadKind {
|
||||
/// A normal non-trivia child has been found.
|
||||
Normal,
|
||||
/// An unbounded child has been found. The boolean indicates whether it was
|
||||
/// on the current line, in which case adding it to the reparsing range is
|
||||
/// compulsory.
|
||||
Unbounded(bool),
|
||||
}
|
||||
|
||||
impl Ahead {
|
||||
fn new(pos: NodePos, at_start: bool, bounded: bool) -> Self {
|
||||
Self {
|
||||
pos,
|
||||
at_start,
|
||||
kind: if bounded { AheadKind::Normal } else { AheadKind::Unbounded(true) },
|
||||
}
|
||||
}
|
||||
|
||||
fn newline(&mut self) {
|
||||
if let AheadKind::Unbounded(current_line) = &mut self.kind {
|
||||
*current_line = false;
|
||||
}
|
||||
}
|
||||
|
||||
fn is_compulsory(self) -> bool {
|
||||
matches!(self.kind, AheadKind::Unbounded(true))
|
||||
}
|
||||
}
|
||||
|
||||
/// Which reparse function to choose for a span of elements.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum ReparseMode {
|
||||
/// Reparse a code block, including its braces.
|
||||
Code,
|
||||
/// Reparse a content block, including its square brackets.
|
||||
Content,
|
||||
/// Reparse elements of the markup. Also specified the initial `at_start`
|
||||
/// state for the reparse and the minimum indent of the reparsed nodes.
|
||||
MarkupElements { at_start: bool, min_indent: usize },
|
||||
}
|
||||
|
||||
/// Whether changes _inside_ this node are safely encapsulated, so that only
|
||||
/// this node must be reparsed.
|
||||
fn is_bounded(kind: SyntaxKind) -> bool {
|
||||
matches!(
|
||||
kind,
|
||||
SyntaxKind::CodeBlock
|
||||
| SyntaxKind::ContentBlock
|
||||
| SyntaxKind::Linebreak
|
||||
| SyntaxKind::SmartQuote
|
||||
| SyntaxKind::BlockComment
|
||||
| SyntaxKind::Space { .. }
|
||||
| SyntaxKind::Escape
|
||||
| SyntaxKind::Shorthand
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether `at_start` would still be true after this node given the
|
||||
/// previous value of the property.
|
||||
fn next_at_start(kind: SyntaxKind, prev: bool) -> bool {
|
||||
match kind {
|
||||
SyntaxKind::Space { newlines: (1..) } => true,
|
||||
SyntaxKind::Space { .. } | SyntaxKind::LineComment | SyntaxKind::BlockComment => {
|
||||
prev
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[rustfmt::skip]
|
||||
mod tests {
|
||||
use std::fmt::Debug;
|
||||
|
||||
use super::*;
|
||||
use super::super::{parse, Source};
|
||||
|
||||
#[track_caller]
|
||||
fn check<T>(text: &str, found: T, expected: T)
|
||||
where
|
||||
T: Debug + PartialEq,
|
||||
{
|
||||
if found != expected {
|
||||
println!("source: {text:?}");
|
||||
println!("expected: {expected:#?}");
|
||||
println!("found: {found:#?}");
|
||||
panic!("test failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn test(prev: &str, range: Range<usize>, with: &str, goal: Range<usize>) {
|
||||
let mut source = Source::detached(prev);
|
||||
let range = source.edit(range, with);
|
||||
check(source.text(), source.root(), &parse(source.text()));
|
||||
assert_eq!(range, goal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_incremental_simple_replacements() {
|
||||
test("hello world", 7 .. 12, "walkers", 0 .. 14);
|
||||
test("some content", 0..12, "", 0..0);
|
||||
test("", 0..0, "do it", 0..5);
|
||||
test("a d e", 1 .. 3, " b c d", 0 .. 9);
|
||||
test("*~ *", 2..2, "*", 0..5);
|
||||
test("_1_\n2a\n3", 5..5, "4", 4..7);
|
||||
test("_1_\n2a\n3~", 8..8, "4", 4..10);
|
||||
test("_1_ 2 3a\n4", 7..7, "5", 0..9);
|
||||
test("* {1+2} *", 5..6, "3", 2..7);
|
||||
test("a #f() e", 1 .. 6, " b c d", 0 .. 9);
|
||||
test("a\nb\nc\nd\ne\n", 5 .. 5, "c", 2 .. 7);
|
||||
test("a\n\nb\n\nc\n\nd\n\ne\n", 7 .. 7, "c", 3 .. 10);
|
||||
test("a\nb\nc *hel a b lo* d\nd\ne", 13..13, "c ", 4..20);
|
||||
test("~~ {a} ~~", 4 .. 5, "b", 3 .. 6);
|
||||
test("{(0, 1, 2)}", 5 .. 6, "11pt", 0..14);
|
||||
test("\n= A heading", 4 .. 4, "n evocative", 0 .. 23);
|
||||
test("for~your~thing", 9 .. 9, "a", 0 .. 15);
|
||||
test("a your thing a", 6 .. 7, "a", 0 .. 14);
|
||||
test("{call(); abc}", 7 .. 7, "[]", 0 .. 15);
|
||||
test("#call() abc", 7 .. 7, "[]", 0 .. 10);
|
||||
test("hi[\n- item\n- item 2\n - item 3]", 11 .. 11, " ", 2 .. 35);
|
||||
test("hi\n- item\nno item\n - item 3", 10 .. 10, "- ", 3..19);
|
||||
test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 16 .. 20, "none", 0..99);
|
||||
test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 33 .. 42, "[_gronk_]", 33..42);
|
||||
test("#grid(columns: (auto, 1fr, 40%), [*plonk*], rect(width: 100%, height: 1pt, fill: conifer), [thing])", 34 .. 41, "_bar_", 33 .. 40);
|
||||
test("{let i=1; for x in range(5) {i}}", 6 .. 6, " ", 0 .. 33);
|
||||
test("{let i=1; for x in range(5) {i}}", 13 .. 14, " ", 0 .. 33);
|
||||
test("hello~~{x}", 7 .. 10, "#f()", 0 .. 11);
|
||||
test("this~is -- in my opinion -- spectacular", 8 .. 10, "---", 0 .. 25);
|
||||
test("understanding `code` is complicated", 15 .. 15, "C ", 0 .. 22);
|
||||
test("{ let x = g() }", 10 .. 12, "f(54", 0 .. 17);
|
||||
test(r#"a ```typst hello``` b"#, 16 .. 17, "", 0 .. 18);
|
||||
test(r#"a ```typst hello```"#, 16 .. 17, "", 0 .. 18);
|
||||
test("#for", 4 .. 4, "//", 0 .. 6);
|
||||
test("#show f: a => b..", 16..16, "c", 0..18);
|
||||
test("a\n#let \nb", 7 .. 7, "i", 2 .. 9);
|
||||
test("a\n#for i \nb", 9 .. 9, "in", 2 .. 12);
|
||||
test("a~https://fun/html", 13..14, "n", 0..18);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_incremental_whitespace_invariants() {
|
||||
test("hello \\ world", 7 .. 8, "a ", 0 .. 14);
|
||||
test("hello \\ world", 7 .. 8, " a", 0 .. 14);
|
||||
test("x = y", 1 .. 1, " + y", 0 .. 6);
|
||||
test("x = y", 1 .. 1, " + y\n", 0 .. 7);
|
||||
test("abc\n= a heading\njoke", 3 .. 4, "\nmore\n\n", 0 .. 21);
|
||||
test("abc\n= a heading\njoke", 3 .. 4, "\nnot ", 0 .. 19);
|
||||
test("#let x = (1, 2 + ;~ Five\r\n\r", 20 .. 23, "2.", 0 .. 23);
|
||||
test("hey #myfriend", 4 .. 4, "\\", 0 .. 14);
|
||||
test("hey #myfriend", 4 .. 4, "\\", 0 .. 6);
|
||||
test("= foo\nbar\n - a\n - b", 6 .. 9, "", 0 .. 11);
|
||||
test("= foo\n bar\n baz", 6 .. 8, "", 0 .. 9);
|
||||
test(" // hi", 1 .. 1, " ", 0 .. 7);
|
||||
test("- \nA", 2..3, "", 0..3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_incremental_type_invariants() {
|
||||
test("a #for x in array {x}", 18 .. 21, "[#x]", 0 .. 22);
|
||||
test("a #let x = 1 {5}", 3 .. 6, "if", 0 .. 11);
|
||||
test("a {let x = 1 {5}} b", 3 .. 6, "if", 2 .. 16);
|
||||
test("#let x = 1 {5}", 4 .. 4, " if", 0 .. 13);
|
||||
test("{let x = 1 {5}}", 4 .. 4, " if", 0 .. 18);
|
||||
test("a // b c #f()", 3 .. 4, "", 0 .. 12);
|
||||
test("{\nf()\n//g(a)\n}", 6 .. 8, "", 0 .. 12);
|
||||
test("a{\nf()\n//g(a)\n}b", 7 .. 9, "", 1 .. 13);
|
||||
test("a #while x {\n g(x) \n} b", 11 .. 11, "//", 0 .. 26);
|
||||
test("{(1, 2)}", 1 .. 1, "while ", 0 .. 14);
|
||||
test("a b c", 1 .. 1, "{[}", 0 .. 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_incremental_wrongly_or_unclosed_things() {
|
||||
test(r#"{"hi"}"#, 4 .. 5, "c", 0 .. 6);
|
||||
test(r"this \u{abcd}", 8 .. 9, "", 0 .. 12);
|
||||
test(r"this \u{abcd} that", 12 .. 13, "", 0 .. 17);
|
||||
test(r"{{let x = z}; a = 1} b", 6 .. 6, "//", 0 .. 24);
|
||||
test("a b c", 1 .. 1, " /* letters */", 0 .. 19);
|
||||
test("a b c", 1 .. 1, " /* letters", 0 .. 16);
|
||||
test("{if i==1 {a} else [b]; b()}", 12 .. 12, " /* letters */", 0 .. 41);
|
||||
test("{if i==1 {a} else [b]; b()}", 12 .. 12, " /* letters", 0 .. 38);
|
||||
test("~~~~", 2 .. 2, "[]", 0 .. 5);
|
||||
test("a[]b", 2 .. 2, "{", 1 .. 4);
|
||||
test("[hello]", 2 .. 3, "]", 0 .. 7);
|
||||
test("{a}", 1 .. 2, "b", 0 .. 3);
|
||||
test("{ a; b; c }", 5 .. 6, "[}]", 0 .. 13);
|
||||
test("#a()\n~", 3..4, "{}", 0..7);
|
||||
test("[]\n~", 1..2, "#if i==0 {true}", 0..18);
|
||||
}
|
||||
}
|
||||
262
src/syntax/reparser.rs
Normal file
262
src/syntax/reparser.rs
Normal file
@ -0,0 +1,262 @@
|
||||
use std::ops::Range;
|
||||
|
||||
use super::{
|
||||
is_newline, parse, reparse_block, reparse_markup, Span, SyntaxKind, SyntaxNode,
|
||||
};
|
||||
|
||||
/// Refresh the given syntax node with as little parsing as possible.
|
||||
///
|
||||
/// Takes the new text, the range in the old text that was replaced and the
|
||||
/// length of the replacement and returns the range in the new text that was
|
||||
/// ultimately reparsed.
|
||||
///
|
||||
/// The high-level API for this function is
|
||||
/// [`Source::edit`](super::Source::edit).
|
||||
pub fn reparse(
|
||||
root: &mut SyntaxNode,
|
||||
text: &str,
|
||||
replaced: Range<usize>,
|
||||
replacement_len: usize,
|
||||
) -> Range<usize> {
|
||||
try_reparse(text, replaced, replacement_len, None, root, 0).unwrap_or_else(|| {
|
||||
let id = root.span().source();
|
||||
*root = parse(text);
|
||||
root.numberize(id, Span::FULL).unwrap();
|
||||
0..text.len()
|
||||
})
|
||||
}
|
||||
|
||||
/// Try to reparse inside the given node.
|
||||
fn try_reparse(
|
||||
text: &str,
|
||||
replaced: Range<usize>,
|
||||
replacement_len: usize,
|
||||
parent_kind: Option<SyntaxKind>,
|
||||
node: &mut SyntaxNode,
|
||||
offset: usize,
|
||||
) -> Option<Range<usize>> {
|
||||
// The range of children which overlap with the edit.
|
||||
let mut overlap = usize::MAX..0;
|
||||
let mut cursor = offset;
|
||||
let node_kind = node.kind();
|
||||
|
||||
for (i, child) in node.children_mut().iter_mut().enumerate() {
|
||||
let prev_range = cursor..cursor + child.len();
|
||||
let prev_len = child.len();
|
||||
let prev_desc = child.descendants();
|
||||
|
||||
// Does the child surround the edit?
|
||||
// If so, try to reparse within it or itself.
|
||||
if !child.is_leaf() && includes(&prev_range, &replaced) {
|
||||
let new_len = prev_len + replacement_len - replaced.len();
|
||||
let new_range = cursor..cursor + new_len;
|
||||
|
||||
// Try to reparse within the child.
|
||||
if let Some(range) = try_reparse(
|
||||
text,
|
||||
replaced.clone(),
|
||||
replacement_len,
|
||||
Some(node_kind),
|
||||
child,
|
||||
cursor,
|
||||
) {
|
||||
assert_eq!(child.len(), new_len);
|
||||
let new_desc = child.descendants();
|
||||
node.update_parent(prev_len, new_len, prev_desc, new_desc);
|
||||
return Some(range);
|
||||
}
|
||||
|
||||
// If the child is a block, try to reparse the block.
|
||||
if child.kind().is_block() {
|
||||
if let Some(newborn) = reparse_block(text, new_range.clone()) {
|
||||
return node
|
||||
.replace_children(i..i + 1, vec![newborn])
|
||||
.is_ok()
|
||||
.then(|| new_range);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Does the child overlap with the edit?
|
||||
if overlaps(&prev_range, &replaced) {
|
||||
overlap.start = overlap.start.min(i);
|
||||
overlap.end = i + 1;
|
||||
}
|
||||
|
||||
// Is the child beyond the edit?
|
||||
if replaced.end < cursor {
|
||||
break;
|
||||
}
|
||||
|
||||
cursor += child.len();
|
||||
}
|
||||
|
||||
// Try to reparse a range of markup expressions within markup. This is only
|
||||
// possible if the markup is top-level or contained in a block, not if it is
|
||||
// contained in things like headings or lists because too much can go wrong
|
||||
// with indent and line breaks.
|
||||
if node.kind() == SyntaxKind::Markup
|
||||
&& (parent_kind == None || parent_kind == Some(SyntaxKind::ContentBlock))
|
||||
&& !overlap.is_empty()
|
||||
{
|
||||
// Add one node of slack in both directions.
|
||||
let children = node.children_mut();
|
||||
let mut start = overlap.start.saturating_sub(1);
|
||||
let mut end = (overlap.end + 1).min(children.len());
|
||||
|
||||
// Expand to the left.
|
||||
while start > 0 && expand(&children[start]) {
|
||||
start -= 1;
|
||||
}
|
||||
|
||||
// Expand to the right.
|
||||
while end < children.len() && expand(&children[end]) {
|
||||
end += 1;
|
||||
}
|
||||
|
||||
// Synthesize what `at_start` would be at the start of the reparse.
|
||||
let mut prefix_len = 0;
|
||||
let mut at_start = true;
|
||||
for child in &children[..start] {
|
||||
prefix_len += child.len();
|
||||
next_at_start(child, &mut at_start);
|
||||
}
|
||||
|
||||
// Determine what `at_start` will have to be at the end of the reparse.
|
||||
let mut prev_len = 0;
|
||||
let mut prev_at_start_after = at_start;
|
||||
for child in &children[start..end] {
|
||||
prev_len += child.len();
|
||||
next_at_start(child, &mut prev_at_start_after);
|
||||
}
|
||||
|
||||
let shifted = offset + prefix_len;
|
||||
let new_len = prev_len + replacement_len - replaced.len();
|
||||
let new_range = shifted..shifted + new_len;
|
||||
let stop_kind = match parent_kind {
|
||||
Some(_) => SyntaxKind::RightBracket,
|
||||
None => SyntaxKind::Eof,
|
||||
};
|
||||
|
||||
if let Some(newborns) =
|
||||
reparse_markup(text, new_range.clone(), &mut at_start, |kind| {
|
||||
kind == stop_kind
|
||||
})
|
||||
{
|
||||
if at_start == prev_at_start_after {
|
||||
return node
|
||||
.replace_children(start..end, newborns)
|
||||
.is_ok()
|
||||
.then(|| new_range);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether the inner range is fully contained in the outer one (no touching).
|
||||
fn includes(outer: &Range<usize>, inner: &Range<usize>) -> bool {
|
||||
outer.start < inner.start && outer.end > inner.end
|
||||
}
|
||||
|
||||
/// Whether the first and second range overlap or touch.
|
||||
fn overlaps(first: &Range<usize>, second: &Range<usize>) -> bool {
|
||||
(first.start <= second.start && second.start <= first.end)
|
||||
|| (second.start <= first.start && first.start <= second.end)
|
||||
}
|
||||
|
||||
/// Whether the selection should be expanded beyond a node of this kind.
|
||||
fn expand(node: &SyntaxNode) -> bool {
|
||||
let kind = node.kind();
|
||||
kind.is_trivia()
|
||||
|| kind.is_error()
|
||||
|| kind == SyntaxKind::Semicolon
|
||||
|| node.text() == "/"
|
||||
|| node.text() == ":"
|
||||
}
|
||||
|
||||
/// Whether `at_start` would still be true after this node given the
|
||||
/// previous value of the property.
|
||||
fn next_at_start(node: &SyntaxNode, at_start: &mut bool) {
|
||||
if node.kind().is_trivia() {
|
||||
if node.text().chars().any(is_newline) {
|
||||
*at_start = true;
|
||||
}
|
||||
} else {
|
||||
*at_start = false;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::ops::Range;
|
||||
|
||||
use super::super::{parse, Source};
|
||||
|
||||
#[track_caller]
|
||||
fn test(prev: &str, range: Range<usize>, with: &str, incremental: bool) {
|
||||
let mut source = Source::detached(prev);
|
||||
let prev = source.root().clone();
|
||||
let range = source.edit(range, with);
|
||||
let found = source.root();
|
||||
let expected = parse(source.text());
|
||||
if found != &expected {
|
||||
eprintln!("source: {:?}", source.text());
|
||||
eprintln!("previous: {prev:#?}");
|
||||
eprintln!("expected: {expected:#?}");
|
||||
eprintln!("found: {found:#?}");
|
||||
panic!("test failed");
|
||||
}
|
||||
if incremental {
|
||||
assert_ne!(source.len_bytes(), range.len());
|
||||
} else {
|
||||
assert_eq!(source.len_bytes(), range.len());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reparse_markup() {
|
||||
test("abc~def~ghi", 5..6, "+", true);
|
||||
test("~~~~~~~", 3..4, "A", true);
|
||||
test("abc~~", 1..2, "", true);
|
||||
test("#var;hello", 9..10, "a", false);
|
||||
test("https:/world", 7..7, "/", false);
|
||||
test("hello world", 7..12, "walkers", false);
|
||||
test("some content", 0..12, "", false);
|
||||
test("", 0..0, "do it", false);
|
||||
test("a d e", 1..3, " b c d", false);
|
||||
test("~*~*~", 2..2, "*", false);
|
||||
test("::1\n2. a\n3", 7..7, "4", true);
|
||||
test("* {1+2} *", 5..6, "3", true);
|
||||
test("{(0, 1, 2)}", 5..6, "11pt", false);
|
||||
test("\n= A heading", 4..4, "n evocative", false);
|
||||
test("#call() abc~d", 7..7, "[]", true);
|
||||
test("a your thing a", 6..7, "a", false);
|
||||
test("#grid(columns: (auto, 1fr, 40%))", 16..20, "4pt", false);
|
||||
test("abc\n= a heading\njoke", 3..4, "\nmore\n\n", true);
|
||||
test("#show f: a => b..", 16..16, "c", false);
|
||||
test("#for", 4..4, "//", false);
|
||||
test("a\n#let \nb", 7..7, "i", true);
|
||||
test("#let x = (1, 2 + ;~ Five\r\n\r", 20..23, "2.", true);
|
||||
test(r"{{let x = z}; a = 1} b", 6..6, "//", false);
|
||||
test(r#"a ```typst hello```"#, 16..17, "", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reparse_block() {
|
||||
test("Hello { x + 1 }!", 8..9, "abc", true);
|
||||
test("A{}!", 2..2, "\"", false);
|
||||
test("{ [= x] }!", 4..4, "=", true);
|
||||
test("[[]]", 2..2, "\\", false);
|
||||
test("[[ab]]", 3..4, "\\", false);
|
||||
test("{}}", 1..1, "{", false);
|
||||
test("A: [BC]", 5..5, "{", false);
|
||||
test("A: [BC]", 5..5, "{}", true);
|
||||
test("{\"ab\"}A", 4..4, "c", true);
|
||||
test("{\"ab\"}A", 4..5, "c", false);
|
||||
test("a[]b", 2..2, "{", false);
|
||||
test("a{call(); abc}b", 7..7, "[]", true);
|
||||
test("a #while x {\n g(x) \n} b", 12..12, "//", true);
|
||||
}
|
||||
}
|
||||
@ -9,8 +9,7 @@ use comemo::Prehashed;
|
||||
use unscanny::Scanner;
|
||||
|
||||
use super::ast::Markup;
|
||||
use super::reparse::reparse;
|
||||
use super::{is_newline, parse, Span, SyntaxNode};
|
||||
use super::{is_newline, parse, reparse, Span, SyntaxNode};
|
||||
use crate::diag::SourceResult;
|
||||
use crate::util::{PathExt, StrExt};
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 24 KiB After Width: | Height: | Size: 24 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 20 KiB |
@ -43,6 +43,7 @@ _Shopping list_
|
||||
- A with 1 tab
|
||||
- B with 2 tabs
|
||||
|
||||
---
|
||||
// This doesn't work because of mixed tabs and spaces.
|
||||
- A with 2 spaces
|
||||
- B with 2 tabs
|
||||
|
||||
@ -208,17 +208,16 @@
|
||||
// Error: 3 expected closing paren
|
||||
{(}
|
||||
|
||||
// Error: 2-3 expected expression, found closing paren
|
||||
// Error: 2-3 unexpected closing paren
|
||||
{)}
|
||||
|
||||
// Error: 4 expected comma
|
||||
// Error: 4-6 unexpected end of block comment
|
||||
{(1*/2)}
|
||||
|
||||
// Error: 6-8 invalid number suffix
|
||||
{(1, 1u 2)}
|
||||
|
||||
// Error: 3-4 expected expression, found comma
|
||||
// Error: 3-4 unexpected comma
|
||||
{(,1)}
|
||||
|
||||
// Missing expression makes named pair incomplete, making this an empty array.
|
||||
|
||||
@ -126,10 +126,12 @@
|
||||
|
||||
// Should output `3`.
|
||||
{
|
||||
// Error: 7-10 expected identifier, found string
|
||||
// Error: 6 expected identifier
|
||||
// Error: 10 expected block
|
||||
for "v"
|
||||
|
||||
// Error: 8 expected keyword `in`
|
||||
// Error: 22 expected block
|
||||
for v let z = 1 + 2
|
||||
|
||||
z
|
||||
|
||||
@ -44,7 +44,7 @@
|
||||
}
|
||||
|
||||
---
|
||||
// Error: 28-47 duplicate argument
|
||||
// Error: 28-34 duplicate argument
|
||||
#set text(family: "Arial", family: "Helvetica")
|
||||
|
||||
---
|
||||
@ -70,7 +70,8 @@
|
||||
#f[1](2)
|
||||
|
||||
---
|
||||
// Error: 7-8 expected expression, found colon
|
||||
// Error: 7 expected expression
|
||||
// Error: 8 expected expression
|
||||
#func(:)
|
||||
|
||||
// Error: 10-12 unexpected end of block comment
|
||||
@ -102,5 +103,4 @@
|
||||
|
||||
---
|
||||
// Error: 2:1 expected quote
|
||||
// Error: 2:1 expected closing paren
|
||||
#func("]
|
||||
|
||||
@ -145,6 +145,16 @@
|
||||
test(greet("Typst", whatever: 10))
|
||||
}
|
||||
|
||||
---
|
||||
// Error: 11-12 duplicate parameter
|
||||
#let f(x, x) = none
|
||||
|
||||
---
|
||||
// Error: 14-15 duplicate parameter
|
||||
// Error: 23-24 duplicate parameter
|
||||
// Error: 35-36 duplicate parameter
|
||||
#let f(a, b, a: none, b: none, c, b) = none
|
||||
|
||||
---
|
||||
// Error: 6-16 expected identifier, named pair or argument sink, found keyed pair
|
||||
{(a, "named": b) => none}
|
||||
|
||||
@ -56,11 +56,11 @@
|
||||
#test(dict, (a: 3, b: 1))
|
||||
|
||||
---
|
||||
// Error: 24-32 pair has duplicate key
|
||||
// Error: 24-29 duplicate key
|
||||
{(first: 1, second: 2, first: 3)}
|
||||
|
||||
---
|
||||
// Error: 17-23 pair has duplicate key
|
||||
// Error: 17-20 duplicate key
|
||||
{(a: 1, "b": 2, "a": 3)}
|
||||
|
||||
---
|
||||
@ -72,8 +72,11 @@
|
||||
// Error: 4-5 expected named or keyed pair, found integer
|
||||
// Error: 5 expected comma
|
||||
// Error: 12-16 expected identifier or string, found boolean
|
||||
// Error: 17-18 expected expression, found colon
|
||||
{(:1 b:"", true::)}
|
||||
// Error: 17 expected expression
|
||||
{(:1 b:"", true:)}
|
||||
|
||||
// Error: 3-8 expected identifier or string, found binary expression
|
||||
{(a + b: "hey")}
|
||||
|
||||
---
|
||||
// Error: 3-15 cannot mutate a temporary value
|
||||
|
||||
@ -36,5 +36,6 @@
|
||||
= A
|
||||
|
||||
---
|
||||
// Error: 8-12 expected identifier, found boolean
|
||||
// Error: 8 expected identifier
|
||||
// Error: 8 expected semicolon or line break
|
||||
{false.true}
|
||||
|
||||
@ -94,7 +94,7 @@
|
||||
// Error: 5 expected identifier
|
||||
#for
|
||||
|
||||
// Error: 7 expected identifier
|
||||
// Error: 5 expected identifier
|
||||
#for//
|
||||
|
||||
// Error: 5 expected identifier
|
||||
@ -106,17 +106,18 @@
|
||||
// Error: 10 expected expression
|
||||
#for v in
|
||||
|
||||
// Error: 15 expected body
|
||||
// Error: 15 expected block
|
||||
#for v in iter
|
||||
|
||||
// Error: 5 expected identifier
|
||||
#for
|
||||
v in iter {}
|
||||
|
||||
// Error: 7-10 expected identifier, found string
|
||||
// Error: 6 expected identifier
|
||||
// Error: 10 expected block
|
||||
A#for "v" thing
|
||||
|
||||
// Error: 6-9 expected identifier, found string
|
||||
// Error: 5 expected identifier
|
||||
#for "v" in iter {}
|
||||
|
||||
// Error: 7 expected keyword `in`
|
||||
|
||||
@ -112,7 +112,7 @@
|
||||
// Error: 4 expected expression
|
||||
{if}
|
||||
|
||||
// Error: 6 expected body
|
||||
// Error: 6 expected block
|
||||
#if x
|
||||
|
||||
// Error: 1-6 unexpected keyword `else`
|
||||
@ -124,11 +124,11 @@
|
||||
x {}
|
||||
|
||||
// Should output `something`.
|
||||
// Error: 6 expected body
|
||||
// Error: 6 expected block
|
||||
#if x something
|
||||
|
||||
// Should output `A thing.`
|
||||
// Error: 19 expected body
|
||||
// Error: 19 expected block
|
||||
A#if false {} else thing
|
||||
|
||||
#if a []else [b]
|
||||
|
||||
@ -81,21 +81,16 @@ This is never reached.
|
||||
#import
|
||||
|
||||
---
|
||||
// Error: 26-29 expected identifier, found string
|
||||
// Error: 26-29 unexpected string
|
||||
#import "module.typ": a, "b", c
|
||||
|
||||
---
|
||||
// Error: 22 expected import items
|
||||
#import "module.typ":
|
||||
|
||||
---
|
||||
// Error: 23-24 expected expression, found assignment operator
|
||||
// Error: 24 expected import items
|
||||
// Error: 23-24 unexpected equals sign
|
||||
#import "module.typ": =
|
||||
|
||||
---
|
||||
// An additional trailing comma.
|
||||
// Error: 31-32 expected expression, found comma
|
||||
// Error: 31-32 unexpected comma
|
||||
#import "module.typ": a, b, c,,
|
||||
|
||||
---
|
||||
@ -105,7 +100,7 @@ This is never reached.
|
||||
|
||||
---
|
||||
// A star in the list.
|
||||
// Error: 26-27 expected expression, found star
|
||||
// Error: 26-27 unexpected star
|
||||
#import "module.typ": a, *, b
|
||||
|
||||
---
|
||||
@ -114,5 +109,10 @@ This is never reached.
|
||||
#import "module.typ": *, a
|
||||
|
||||
---
|
||||
// Error: 13-17 expected identifier, found named pair
|
||||
// Error: 14-15 unexpected colon
|
||||
// Error: 16-17 unexpected integer
|
||||
#import "": a: 1
|
||||
|
||||
---
|
||||
// Error: 14 expected comma
|
||||
#import "": a b
|
||||
|
||||
@ -39,7 +39,8 @@ Three
|
||||
// Error: 5 expected identifier
|
||||
{let}
|
||||
|
||||
// Error: 6-9 expected identifier, found string
|
||||
// Error: 5 expected identifier
|
||||
// Error: 5 expected semicolon or line break
|
||||
#let "v"
|
||||
|
||||
// Error: 7 expected semicolon or line break
|
||||
@ -48,7 +49,8 @@ Three
|
||||
// Error: 9 expected expression
|
||||
#let v =
|
||||
|
||||
// Error: 6-9 expected identifier, found string
|
||||
// Error: 5 expected identifier
|
||||
// Error: 5 expected semicolon or line break
|
||||
#let "v" = 1
|
||||
|
||||
// Terminated because expression ends.
|
||||
@ -61,7 +63,7 @@ Three
|
||||
#let v5 = (1, 2 + ; Five
|
||||
|
||||
---
|
||||
// Error: 13 expected body
|
||||
// Error: 13 expected equals sign
|
||||
#let func(x)
|
||||
|
||||
// Error: 15 expected expression
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
#min(.."nope")
|
||||
|
||||
---
|
||||
// Error: 8-14 expected identifier, named pair or argument sink, found spread
|
||||
// Error: 10-14 expected identifier, found boolean
|
||||
#let f(..true) = none
|
||||
|
||||
---
|
||||
|
||||
@ -49,12 +49,12 @@
|
||||
// Error: 7 expected expression
|
||||
{while}
|
||||
|
||||
// Error: 9 expected body
|
||||
// Error: 9 expected block
|
||||
#while x
|
||||
|
||||
// Error: 7 expected expression
|
||||
#while
|
||||
x {}
|
||||
|
||||
// Error: 9 expected body
|
||||
// Error: 9 expected block
|
||||
#while x something
|
||||
|
||||
@ -38,6 +38,6 @@ _Hello
|
||||
World
|
||||
|
||||
---
|
||||
// Error: 1:12 expected star
|
||||
// Error: 2:1 expected star
|
||||
_Cannot *be_ interleaved*
|
||||
// Error: 25 expected star
|
||||
// Error: 25 expected underscore
|
||||
[_Cannot *be interleaved]
|
||||
|
||||
@ -41,7 +41,8 @@ The keyword ```rust let```.
|
||||
// First line is not dedented and leading space is still possible.
|
||||
``` A
|
||||
B
|
||||
C```
|
||||
C
|
||||
```
|
||||
|
||||
---
|
||||
// Unterminated.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user