konevsa/typst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7 from typst/parser-update

Browse Source 
Parser update
This commit is contained in:
Laurenz 2020-07-29 17:46:57 +02:00 committed by GitHub
commit f34ba3dcda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 840 additions and 891 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/func.rs
View File

@ -1,9 +1,8 @@
//! Trait and prelude for custom functions.
use crate::Pass;
use crate::syntax::ParseContext;
use crate::syntax::func::FuncHeader;
use crate::syntax::span::Spanned;
use crate::syntax::ParseState;
use crate::syntax::func::FuncCall;
/// Types that are useful for creating your own functions.
pub mod prelude {
@ -17,7 +16,6 @@ pub mod prelude {
pub use crate::syntax::span::{Span, Spanned};
}
/// Parse a function from source code.
pub trait ParseFunc {
/// A metadata type whose value is passed into the function parser. This
@ -33,9 +31,8 @@ pub trait ParseFunc {
/// Parse the header and body into this function given a context.
fn parse(
header: FuncHeader,
body: Option<Spanned<&str>>,
ctx: ParseContext,
header: FuncCall,
state: &ParseState,
metadata: Self::Meta,
) -> Pass<Self> where Self: Sized;
}
@ -53,8 +50,8 @@ pub trait ParseFunc {
/// body: Option<SyntaxModel>,
/// }
///
/// parse(header, body, ctx, f) {
/// let body = body!(opt: body, ctx, f);
/// parse(header, body, state, f) {
/// let body = body!(opt: body, state, f);
/// let hidden = header.args.pos.get::<bool>(&mut f.problems)
/// .or_missing(&mut f.problems, header.name.span, "hidden")
/// .unwrap_or(false);
@ -112,7 +109,7 @@ macro_rules! function {
(@parse($name:ident, $meta:ty) parse(
$header:ident,
$body:ident,
$ctx:ident,
$state:ident,
$feedback:ident,
$metadata:ident
) $code:block $($r:tt)*) => {
@ -120,18 +117,18 @@ macro_rules! function {
type Meta = $meta;
fn parse(
#[allow(unused)] mut header: $crate::syntax::func::FuncHeader,
#[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>,
#[allow(unused)] $ctx: $crate::syntax::ParseContext,
#[allow(unused)] mut call: $crate::syntax::func::FuncCall,
#[allow(unused)] $state: &$crate::syntax::ParseState,
#[allow(unused)] $metadata: Self::Meta,
) -> $crate::Pass<Self> where Self: Sized {
let mut feedback = $crate::Feedback::new();
#[allow(unused)] let $header = &mut header;
#[allow(unused)] let $header = &mut call.header;
#[allow(unused)] let $body = &mut call.body;
#[allow(unused)] let $feedback = &mut feedback;
let func = $code;
for arg in header.args.into_iter() {
for arg in call.header.args.into_iter() {
error!(@feedback, arg.span, "unexpected argument");
}
@ -167,21 +164,20 @@ macro_rules! function {
/// Parse the body of a function.
///
/// - If the function does not expect a body, use `body!(nope: body, feedback)`.
/// - If the function can have a body, use `body!(opt: body, ctx, feedback,
/// - If the function can have a body, use `body!(opt: body, state, feedback,
/// decos)`.
///
/// # Arguments
/// - The `$body` should be of type `Option<Spanned<&str>>`.
/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for
/// parsing.
/// - The `$state` is the parse state to use.
/// - The `$feedback` should be a mutable references to a
/// [`Feedback`](crate::Feedback) struct which is filled with the feedback
/// information arising from parsing.
#[macro_export]
macro_rules! body {
(opt: $body:expr, $ctx:expr, $feedback:expr) => ({
(opt: $body:expr, $state:expr, $feedback:expr) => ({
$body.map(|body| {
let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
let parsed = $crate::syntax::parse(body.v, body.span.start, $state);
$feedback.extend(parsed.feedback);
parsed.output
})

2
src/layout/model.rs
View File

@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> {
}).await;
// Add the errors generated by the model to the error list.
self.feedback.extend_offset(model.span.start, layouted.feedback);
self.feedback.extend_offset(layouted.feedback, model.span.start);
for command in layouted.output {
self.execute_command(command, model.span).await;

21
src/lib.rs
View File

@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor};
use crate::problem::Problems;
use crate::layout::MultiLayout;
use crate::style::{LayoutStyle, PageStyle, TextStyle};
use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse};
use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse};
use crate::syntax::span::{Position, SpanVec, offset_spans};
@ -42,6 +42,8 @@ macro_rules! pub_use_mod {
};
}
#[macro_use]
mod macros;
#[macro_use]
pub mod problem;
pub mod export;
@ -57,14 +59,13 @@ pub mod syntax;
/// Transforms source code into typesetted layouts.
///
/// A typesetter can be configured through various methods.
#[derive(Debug)]
pub struct Typesetter {
/// The font loader shared by all typesetting processes.
loader: GlobalFontLoader,
/// The base layouting style.
style: LayoutStyle,
/// The standard library scope.
scope: Scope,
/// The base parser state.
parse_state: ParseState,
/// Whether to render debug boxes.
debug: bool,
}
@ -84,7 +85,7 @@ impl Typesetter {
Typesetter {
loader: RefCell::new(FontLoader::new(provider)),
style: LayoutStyle::default(),
scope: Scope::with_std(),
parse_state: ParseState { scope: Scope::with_std() },
debug: false,
}
}
@ -111,7 +112,7 @@ impl Typesetter {
/// Parse source code into a syntax tree.
pub fn parse(&self, src: &str) -> Pass<SyntaxModel> {
parse(Position::ZERO, src, ParseContext { scope: &self.scope })
parse(src, Position::ZERO, &self.parse_state)
}
/// Layout a syntax tree and return the produced layout.
@ -203,10 +204,10 @@ impl Feedback {
}
/// Add more feedback whose spans are local and need to be offset by an
/// `offset` to be correct for this feedbacks context.
pub fn extend_offset(&mut self, offset: Position, other: Feedback) {
self.problems.extend(offset_spans(offset, other.problems));
self.decos.extend(offset_spans(offset, other.decos));
/// `offset` to be correct in this feedback's context.
pub fn extend_offset(&mut self, more: Feedback, offset: Position) {
self.problems.extend(offset_spans(more.problems, offset));
self.decos.extend(offset_spans(more.decos, offset));
}
}

4
src/library/mod.rs
View File

@ -53,10 +53,10 @@ function! {
body: Option<SyntaxModel>,
}
parse(header, body, ctx, f) {
parse(header, body, state, f) {
header.args.pos.items.clear();
header.args.key.pairs.clear();
ValFunc { body: body!(opt: body, ctx, f) }
ValFunc { body: body!(opt: body, state, f) }
}
layout(self, ctx, f) {

4
src/library/page.rs
View File

@ -12,7 +12,7 @@ function! {
flip: bool,
}
parse(header, body, ctx, f) {
parse(header, body, state, f) {
body!(nope: body, f);
PageSizeFunc {
paper: header.args.pos.get::<Paper>(&mut f.problems),
@ -50,7 +50,7 @@ function! {
padding: PaddingMap,
}
parse(header, body, ctx, f) {
parse(header, body, state, f) {
body!(nope: body, f);
PageMarginsFunc {
padding: PaddingMap::parse(&mut f.problems, &mut header.args),

6
src/library/spacing.rs
View File

@ -46,9 +46,9 @@ function! {
type Meta = ContentKind;
parse(header, body, ctx, f, meta) {
parse(header, body, state, f, meta) {
ContentSpacingFunc {
body: body!(opt: body, ctx, f),
body: body!(opt: body, state, f),
content: meta,
spacing: header.args.pos.get::<f64>(&mut f.problems)
.map(|num| num as f32)
@ -84,7 +84,7 @@ function! {
type Meta = Option<SpecificAxis>;
parse(header, body, ctx, f, meta) {
parse(header, body, state, f, meta) {
body!(nope: body, f);
SpacingFunc {
spacing: if let Some(axis) = meta {

25
src/macros.rs Normal file
View File

@ -0,0 +1,25 @@
#![allow(unused)]
/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`.
/// This fits use cases the `?`-operator does not cover, like:
/// ```
/// try_or!(result, continue);
/// ```
macro_rules! try_or {
($result:expr, $or:expr $(,)?) => {
match $result {
Ok(v) => v,
Err(_) => { $or }
}
};
}
/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`.
macro_rules! try_opt_or {
($option:expr, $or:expr $(,)?) => {
match $option {
Some(v) => v,
None => { $or }
}
};
}

6
src/syntax/expr.rs
View File

@ -93,9 +93,6 @@ impl Debug for Expr {
/// A unicode identifier.
///
/// The identifier must be valid! This is checked in [`Ident::new`] or
/// [`is_identifier`].
///
/// # Example
/// ```typst
/// [func: "hi", ident]
@ -105,7 +102,8 @@ impl Debug for Expr {
pub struct Ident(pub String);
impl Ident {
/// Create a new identifier from a string checking that it is valid.
/// Create a new identifier from a string checking that it is a valid
/// unicode identifier.
pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> {
if is_identifier(ident.as_ref()) {
Some(Ident(ident.into()))

17
src/syntax/func/mod.rs
View File

@ -9,22 +9,23 @@ pub_use_mod!(maps);
pub_use_mod!(keys);
pub_use_mod!(values);
/// An invocation of a function.
#[derive(Debug, Clone, PartialEq)]
pub struct FuncCall<'s> {
pub header: FuncHeader,
/// The body as a raw string containing what's inside of the brackets.
pub body: Option<Spanned<&'s str>>,
}
/// The parsed header of a function.
/// The parsed header of a function (everything in the first set of brackets).
#[derive(Debug, Clone, PartialEq)]
pub struct FuncHeader {
/// The function name, that is:
/// ```typst
/// [box: w=5cm]
/// ^^^
/// ```
pub name: Spanned<Ident>,
/// The arguments passed to the function.
pub args: FuncArgs,
}
/// The positional and keyword arguments passed to a function.
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Default, Clone, PartialEq)]
pub struct FuncArgs {
/// The positional arguments.
pub pos: Tuple,

1172
src/syntax/parsing.rs
View File

File diff suppressed because it is too large Load Diff

22
src/syntax/scope.rs
View File

@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter};
use crate::Pass;
use crate::func::ParseFunc;
use super::func::FuncHeader;
use super::parsing::ParseContext;
use super::span::Spanned;
use super::func::FuncCall;
use super::parsing::ParseState;
use super::Model;
/// A map from identifiers to function parsers.
pub struct Scope {
parsers: HashMap<String, Box<Parser>>,
@ -50,10 +48,8 @@ impl Scope {
}
/// Return the parser with the given name if there is one.
pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
self.parsers.get(name)
.map(|x| &**x)
.ok_or_else(|| &*self.fallback)
pub fn get_parser(&self, name: &str) -> Option<&Parser> {
self.parsers.get(name).map(AsRef::as_ref)
}
/// Return the fallback parser.
@ -72,16 +68,12 @@ impl Debug for Scope {
/// A function which parses the source of a function into a model type which
/// implements [`Model`].
type Parser = dyn Fn(
FuncHeader,
Option<Spanned<&str>>,
ParseContext,
) -> Pass<Box<dyn Model>>;
type Parser = dyn Fn(FuncCall, &ParseState) -> Pass<Box<dyn Model>>;
fn parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser>
where F: ParseFunc + Model + 'static {
Box::new(move |h, b, c| {
F::parse(h, b, c, metadata.clone())
Box::new(move |f, s| {
F::parse(f, s, metadata.clone())
.map(|model| Box::new(model) as Box<dyn Model>)
})
}

214
src/syntax/span.rs
View File

@ -4,6 +4,119 @@ use std::fmt::{self, Debug, Formatter};
use std::ops::{Add, Sub};
use serde::Serialize;
/// A vector of spanned values of type `T`.
pub type SpanVec<T> = Vec<Spanned<T>>;
/// [Offset](Span::offset) all spans in a vector of spanned things by a start
/// position.
pub fn offset_spans<T>(
vec: SpanVec<T>,
start: Position,
) -> impl Iterator<Item=Spanned<T>> {
vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
}
/// A value with the span it corresponds to in the source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct Spanned<T> {
/// The value.
pub v: T,
/// The corresponding span.
pub span: Span,
}
impl<T> Spanned<T> {
/// Create a new instance from a value and its span.
pub fn new(v: T, span: Span) -> Spanned<T> {
Spanned { v, span }
}
/// Create a new instance from a value with the zero span.
pub fn zero(v: T) -> Spanned<T> {
Spanned { v, span: Span::ZERO }
}
/// Access the value.
pub fn value(self) -> T {
self.v
}
/// Map the value using a function while keeping the span.
pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
Spanned { v: f(self.v), span: self.span }
}
/// Maps the span while keeping the value.
pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
self.span = f(self.span);
self
}
}
impl<T: Debug> Debug for Spanned<T> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
self.v.fmt(f)?;
if f.alternate() {
f.write_str(" ")?;
self.span.fmt(f)?;
}
Ok(())
}
}
/// Locates a slice of source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct Span {
/// The inclusive start position.
pub start: Position,
/// The inclusive end position.
pub end: Position,
}
impl Span {
/// The zero span.
pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO };
/// Create a new span from start and end positions.
pub fn new(start: Position, end: Position) -> Span {
Span { start, end }
}
/// Create a new span with the earlier start and later end position.
pub fn merge(a: Span, b: Span) -> Span {
Span {
start: a.start.min(b.start),
end: a.end.max(b.end),
}
}
/// Create a span including just a single position.
pub fn at(pos: Position) -> Span {
Span { start: pos, end: pos }
}
/// Expand a span by merging it with another span.
pub fn expand(&mut self, other: Span) {
*self = Span::merge(*self, other)
}
/// Offset a span by a start position.
///
/// This is, for example, used to translate error spans from function local
/// to global.
pub fn offset(self, start: Position) -> Span {
Span {
start: start + self.start,
end: start + self.end,
}
}
}
impl Debug for Span {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "<{:?}-{:?}>", self.start, self.end)
}
}
/// Zero-indexed line-column position in source code.
#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)]
@ -18,7 +131,7 @@ impl Position {
/// The line 0, column 0 position.
pub const ZERO: Position = Position { line: 0, column: 0 };
/// Crete a new instance from line and column.
/// Create a new position from line and column.
pub fn new(line: usize, column: usize) -> Position {
Position { line, column }
}
@ -65,102 +178,3 @@ impl Debug for Position {
write!(f, "{}:{}", self.line, self.column)
}
}
/// Locates a slice of source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct Span {
/// The inclusive start position.
pub start: Position,
/// The inclusive end position.
pub end: Position,
}
impl Span {
/// The zero span.
pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO };
/// Create a new span from start and end positions.
pub fn new(start: Position, end: Position) -> Span {
Span { start, end }
}
/// Create a new span with the earlier start and later end position.
pub fn merge(a: Span, b: Span) -> Span {
Span {
start: a.start.min(b.start),
end: a.end.max(b.end),
}
}
/// Create a span including just a single position.
pub fn at(pos: Position) -> Span {
Span { start: pos, end: pos }
}
/// Offset a span by a start position.
///
/// This is, for example, used to translate error spans from function local
/// to global.
pub fn offset(self, start: Position) -> Span {
Span {
start: start + self.start,
end: start + self.end,
}
}
}
impl Debug for Span {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "({:?} -> {:?})", self.start, self.end)
}
}
/// A value with the span it corresponds to in the source code.
#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
pub struct Spanned<T> {
/// The value.
pub v: T,
/// The corresponding span.
pub span: Span,
}
impl<T> Spanned<T> {
/// Create a new instance from a value and its span.
pub fn new(v: T, span: Span) -> Spanned<T> {
Spanned { v, span }
}
/// Access the value.
pub fn value(self) -> T {
self.v
}
/// Map the value using a function while keeping the span.
pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
Spanned { v: f(self.v), span: self.span }
}
/// Maps the span while keeping the value.
pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
self.span = f(self.span);
self
}
}
impl<T: Debug> Debug for Spanned<T> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
self.v.fmt(f)?;
f.write_str(" ")?;
self.span.fmt(f)
}
}
/// A vector of spanned things.
pub type SpanVec<T> = Vec<Spanned<T>>;
/// [Offset](Span::offset) all spans in a vector of spanned things by a start
/// position.
pub fn offset_spans<T>(start: Position, vec: SpanVec<T>) -> impl Iterator<Item=Spanned<T>> {
vec.into_iter()
.map(move |s| s.map_span(|span| span.offset(start)))
}

68
src/syntax/test.rs
View File

@ -1,17 +1,14 @@
use std::fmt::Debug;
use super::func::FuncHeader;
use super::span::Spanned;
use super::expr::{Expr, Tuple, NamedTuple, Object};
use super::span::{Span, Spanned};
use super::tokens::Token;
use super::*;
/// Check whether the expected and found results for the given source code
/// match by the comparison function, and print them out otherwise.
pub fn check<T>(src: &str, exp: T, found: T, spans: bool)
/// Check whether the expected and found results are the same.
pub fn check<T>(src: &str, exp: T, found: T, cmp_spans: bool)
where T: Debug + PartialEq + SpanlessEq {
let cmp = if spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
let cmp = if cmp_spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
if !cmp(&exp, &found) {
println!("source: {:?}", src);
println!("expected: {:#?}", exp);
@ -23,16 +20,25 @@ where T: Debug + PartialEq + SpanlessEq {
/// Create a vector of optionally spanned expressions from a list description.
///
/// # Examples
/// When you want to add span information to the items, the format is as
/// follows.
/// ```
/// // With spans
/// spanned![(0:0, 0:5, "hello"), (0:5, 0:3, "world")]
///
/// // Without spans: Implicit zero spans.
/// spanned!["hello", "world"]
/// ```
/// The span information can simply be omitted to create a vector with items
/// that are spanned with zero spans.
macro_rules! spanned {
(item ($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
#[allow(unused_imports)]
macro_rules! span_vec {
($(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
(vec![$(span_item!(($sl:$sc, $el:$ec, $v))),*], true)
};
($($v:expr),* $(,)?) => {
(vec![$(span_item!($v)),*], false)
};
}
macro_rules! span_item {
(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
use $crate::syntax::span::{Position, Span, Spanned};
Spanned {
span: Span::new(
@ -43,22 +49,9 @@ macro_rules! spanned {
}
});
(item $v:expr) => {
$crate::syntax::test::zspan($v)
($v:expr) => {
$crate::syntax::span::Spanned::zero($v)
};
(vec $(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
(vec![$(spanned![item ($sl:$sc, $el:$ec, $v)]),*], true)
};
(vec $($v:expr),* $(,)?) => {
(vec![$($crate::syntax::test::zspan($v)),*], false)
};
}
/// Span an element with a zero span.
pub fn zspan<T>(v: T) -> Spanned<T> {
Spanned { v, span: Span::ZERO }
}
function! {
@ -70,13 +63,13 @@ function! {
pub body: Option<SyntaxModel>,
}
parse(header, body, ctx, f) {
parse(header, body, state, f) {
let cloned = header.clone();
header.args.pos.items.clear();
header.args.key.pairs.clear();
DebugFn {
header: cloned,
body: body!(opt: body, ctx, f),
body: body!(opt: body, state, f),
}
}
@ -120,8 +113,8 @@ impl SpanlessEq for DebugFn {
impl SpanlessEq for Expr {
fn spanless_eq(&self, other: &Expr) -> bool {
match (self, other) {
(Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
(Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
(Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
(Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
(Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b),
(Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
@ -175,8 +168,7 @@ impl<T: SpanlessEq> SpanlessEq for Box<T> {
}
}
/// Implement `SpanlessEq` by just forwarding to `PartialEq`.
macro_rules! forward {
macro_rules! impl_through_partial_eq {
($type:ty) => {
impl SpanlessEq for $type {
fn spanless_eq(&self, other: &$type) -> bool {
@ -186,6 +178,8 @@ macro_rules! forward {
};
}
forward!(String);
forward!(Token<'_>);
forward!(Decoration);
impl_through_partial_eq!(Token<'_>);
// Implement for string and decoration to be able to compare feedback.
impl_through_partial_eq!(String);
impl_through_partial_eq!(Decoration);

134
src/syntax/tokens.rs
View File

@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID;
use crate::size::Size;
use super::span::{Position, Span, Spanned};
use self::Token::*;
use self::TokenizationMode::*;
use Token::*;
use TokenMode::*;
/// A minimal semantic entity of source code.
#[derive(Debug, Copy, Clone, PartialEq)]
@ -152,7 +151,7 @@ impl<'s> Token<'s> {
#[derive(Debug)]
pub struct Tokens<'s> {
src: &'s str,
mode: TokenizationMode,
mode: TokenMode,
iter: Peekable<Chars<'s>>,
position: Position,
index: usize,
@ -163,20 +162,22 @@ pub struct Tokens<'s> {
/// backtick tokens.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
#[allow(missing_docs)]
pub enum TokenizationMode {
pub enum TokenMode {
Header,
Body,
}
impl<'s> Tokens<'s> {
/// Create a new token iterator with the given mode where the first token
/// span starts an the given `start` position.
pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
/// Create a new token iterator with the given mode.
///
/// The first token's span starts an the given `offset` position instead of
/// the zero position.
pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> {
Tokens {
src,
mode,
iter: src.chars().peekable(),
position: start,
position: offset,
index: 0,
}
}
@ -188,7 +189,7 @@ impl<'s> Tokens<'s> {
}
/// The line-colunn position in the source at which the last token ends and
/// next token will start. This position is
/// next token will start.
pub fn pos(&self) -> Position {
self.position
}
@ -204,15 +205,15 @@ impl<'s> Iterator for Tokens<'s> {
let token = match first {
// Comments.
'/' if self.peek() == Some('/') => self.parse_line_comment(),
'/' if self.peek() == Some('*') => self.parse_block_comment(),
'/' if self.peek() == Some('/') => self.read_line_comment(),
'/' if self.peek() == Some('*') => self.read_block_comment(),
'*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace.
c if c.is_whitespace() => self.parse_whitespace(start),
c if c.is_whitespace() => self.read_whitespace(start),
// Functions.
'[' => self.parse_function(start),
'[' => self.read_function(start),
']' => Invalid("]"),
// Syntactic elements in function headers.
@ -230,7 +231,7 @@ impl<'s> Iterator for Tokens<'s> {
'/' if self.mode == Header => Slash,
// String values.
'"' if self.mode == Header => self.parse_string(),
'"' if self.mode == Header => self.read_string(),
// Star serves a double purpose as a style modifier
// and a expression operator in the header.
@ -238,13 +239,13 @@ impl<'s> Iterator for Tokens<'s> {
// Style toggles.
'_' if self.mode == Body => Underscore,
'`' if self.mode == Body => self.parse_raw(),
'`' if self.mode == Body => self.read_raw(),
// An escaped thing.
'\\' if self.mode == Body => self.parse_escaped(),
'\\' if self.mode == Body => self.read_escaped(),
// A hex expression.
'#' if self.mode == Header => self.parse_hex_value(),
'#' if self.mode == Header => self.read_hex(),
// Expressions or just strings.
c => {
@ -267,7 +268,7 @@ impl<'s> Iterator for Tokens<'s> {
}, false, -(c.len_utf8() as isize), 0).0;
if self.mode == Header {
self.parse_expr(text)
self.read_expr(text)
} else {
Text(text)
}
@ -282,11 +283,11 @@ impl<'s> Iterator for Tokens<'s> {
}
impl<'s> Tokens<'s> {
fn parse_line_comment(&mut self) -> Token<'s> {
fn read_line_comment(&mut self) -> Token<'s> {
LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
}
fn parse_block_comment(&mut self) -> Token<'s> {
fn read_block_comment(&mut self) -> Token<'s> {
enum Last { Slash, Star, Other }
self.eat();
@ -314,14 +315,14 @@ impl<'s> Tokens<'s> {
}, true, 0, -2).0)
}
fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
fn read_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos();
Space(end.line - start.line)
}
fn parse_function(&mut self, start: Position) -> Token<'s> {
fn read_function(&mut self, start: Position) -> Token<'s> {
let (header, terminated) = self.read_function_part(Header);
self.eat();
@ -341,7 +342,7 @@ impl<'s> Tokens<'s> {
Function { header, body: Some(Spanned { v: body, span }), terminated }
}
fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) {
fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) {
let start = self.index();
let mut terminated = false;
@ -353,11 +354,11 @@ impl<'s> Tokens<'s> {
self.eat();
match n {
'[' => { self.parse_function(Position::ZERO); }
'/' if self.peek() == Some('/') => { self.parse_line_comment(); }
'/' if self.peek() == Some('*') => { self.parse_block_comment(); }
'"' if mode == Header => { self.parse_string(); }
'`' if mode == Body => { self.parse_raw(); }
'[' => { self.read_function(Position::ZERO); }
'/' if self.peek() == Some('/') => { self.read_line_comment(); }
'/' if self.peek() == Some('*') => { self.read_block_comment(); }
'"' if mode == Header => { self.read_string(); }
'`' if mode == Body => { self.read_raw(); }
'\\' => { self.eat(); }
_ => {}
}
@ -367,12 +368,12 @@ impl<'s> Tokens<'s> {
(&self.src[start .. end], terminated)
}
fn parse_string(&mut self) -> Token<'s> {
fn read_string(&mut self) -> Token<'s> {
let (string, terminated) = self.read_until_unescaped('"');
ExprStr { string, terminated }
}
fn parse_raw(&mut self) -> Token<'s> {
fn read_raw(&mut self) -> Token<'s> {
let (raw, terminated) = self.read_until_unescaped('`');
Raw { raw, terminated }
}
@ -390,7 +391,7 @@ impl<'s> Tokens<'s> {
}, true, 0, -1)
}
fn parse_escaped(&mut self) -> Token<'s> {
fn read_escaped(&mut self) -> Token<'s> {
fn is_escapable(c: char) -> bool {
match c {
'[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true,
@ -410,7 +411,7 @@ impl<'s> Tokens<'s> {
}
}
fn parse_hex_value(&mut self) -> Token<'s> {
fn read_hex(&mut self) -> Token<'s> {
// This will parse more than the permissable 0-9, a-f, A-F character
// ranges to provide nicer error messages later.
ExprHex(self.read_string_until(
@ -419,7 +420,7 @@ impl<'s> Tokens<'s> {
).0)
}
fn parse_expr(&mut self, text: &'s str) -> Token<'s> {
fn read_expr(&mut self, text: &'s str) -> Token<'s> {
if let Ok(b) = text.parse::<bool>() {
ExprBool(b)
} else if let Ok(num) = text.parse::<f64>() {
@ -435,8 +436,11 @@ impl<'s> Tokens<'s> {
}
}
/// Will read the input stream until the argument F evaluates to `true`
/// for the current character.
/// Will read the input stream until `f` evaluates to `true`. When
/// `eat_match` is true, the token for which `f` was true is consumed.
/// Returns the string from the index where this was called offset by
/// `offset_start` to the end offset by `offset_end`. The end is before or
/// after the match depending on `eat_match`.
fn read_string_until<F>(
&mut self,
mut f: F,
@ -527,8 +531,8 @@ pub fn is_identifier(string: &str) -> bool {
true
}
#[cfg(test)]
#[allow(non_snake_case)]
mod tests {
use super::super::test::check;
use super::*;
@ -549,31 +553,23 @@ mod tests {
Slash,
};
#[allow(non_snake_case)]
fn Str(string: &'static str, terminated: bool) -> Token<'static> {
Token::ExprStr { string, terminated }
}
#[allow(non_snake_case)]
fn Raw(raw: &'static str, terminated: bool) -> Token<'static> {
Token::Raw { raw, terminated }
}
/// Test whether the given string tokenizes into the given list of tokens.
macro_rules! t {
($mode:expr, $source:expr => [$($tokens:tt)*]) => {
let (exp, spans) = spanned![vec $($tokens)*];
let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>();
let (exp, spans) = span_vec![$($tokens)*];
let found = Tokens::new($source, Position::ZERO, $mode).collect::<Vec<_>>();
check($source, exp, found, spans);
}
}
/// Write down a function token compactly.
fn Str(string: &str, terminated: bool) -> Token { Token::ExprStr { string, terminated } }
fn Raw(raw: &str, terminated: bool) -> Token { Token::Raw { raw, terminated } }
macro_rules! func {
($header:expr, Some($($tokens:tt)*), $terminated:expr) => {
Function {
header: $header,
body: Some(spanned![item $($tokens)*]),
body: Some(span_item!(($($tokens)*))),
terminated: $terminated,
}
};
@ -674,12 +670,12 @@ mod tests {
fn tokenize_functions() {
t!(Body, "a[f]" => [T("a"), func!("f", None, true)]);
t!(Body, "[f]a" => [func!("f", None, true), T("a")]);
t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some((0:4, 0:5, " ")), true)]);
t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some((0:4, 0:5, " ")), true), T("a")]);
t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some(0:4, 0:5, " "), true)]);
t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some(0:4, 0:5, " "), true), T("a")]);
t!(Body, "[f: [=][*]]" => [func!("f: [=][*]", None, true)]);
t!(Body, "[_][[,],]," => [func!("_", Some((0:4, 0:8, "[,],")), true), T(",")]);
t!(Body, "[=][=][=]" => [func!("=", Some((0:4, 0:5, "=")), true), func!("=", None, true)]);
t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:4, 0:13, "[=][=][=]")), true)]);
t!(Body, "[_][[,],]," => [func!("_", Some(0:4, 0:8, "[,],"), true), T(",")]);
t!(Body, "[=][=][=]" => [func!("=", Some(0:4, 0:5, "="), true), func!("=", None, true)]);
t!(Body, "[=][[=][=][=]]" => [func!("=", Some(0:4, 0:13, "[=][=][=]"), true)]);
t!(Header, "[" => [func!("", None, false)]);
t!(Header, "]" => [Invalid("]")]);
}
@ -693,25 +689,25 @@ mod tests {
t!(Body, "[f: `]" => [func!("f: `", None, true)]);
// End of function with strings and carets in bodies
t!(Body, "[f][\"]" => [func!("f", Some((0:4, 0:5, "\"")), true)]);
t!(Body, r#"[f][\"]"# => [func!("f", Some((0:4, 0:6, r#"\""#)), true)]);
t!(Body, "[f][`]" => [func!("f", Some((0:4, 0:6, "`]")), false)]);
t!(Body, "[f][\\`]" => [func!("f", Some((0:4, 0:6, "\\`")), true)]);
t!(Body, "[f][`raw`]" => [func!("f", Some((0:4, 0:9, "`raw`")), true)]);
t!(Body, "[f][`raw]" => [func!("f", Some((0:4, 0:9, "`raw]")), false)]);
t!(Body, "[f][`raw]`]" => [func!("f", Some((0:4, 0:10, "`raw]`")), true)]);
t!(Body, "[f][`\\`]" => [func!("f", Some((0:4, 0:8, "`\\`]")), false)]);
t!(Body, "[f][`\\\\`]" => [func!("f", Some((0:4, 0:8, "`\\\\`")), true)]);
t!(Body, "[f][\"]" => [func!("f", Some(0:4, 0:5, "\""), true)]);
t!(Body, r#"[f][\"]"# => [func!("f", Some(0:4, 0:6, r#"\""#), true)]);
t!(Body, "[f][`]" => [func!("f", Some(0:4, 0:6, "`]"), false)]);
t!(Body, "[f][\\`]" => [func!("f", Some(0:4, 0:6, "\\`"), true)]);
t!(Body, "[f][`raw`]" => [func!("f", Some(0:4, 0:9, "`raw`"), true)]);
t!(Body, "[f][`raw]" => [func!("f", Some(0:4, 0:9, "`raw]"), false)]);
t!(Body, "[f][`raw]`]" => [func!("f", Some(0:4, 0:10, "`raw]`"), true)]);
t!(Body, "[f][`\\`]" => [func!("f", Some(0:4, 0:8, "`\\`]"), false)]);
t!(Body, "[f][`\\\\`]" => [func!("f", Some(0:4, 0:8, "`\\\\`"), true)]);
// End of function with comments
t!(Body, "[f][/*]" => [func!("f", Some((0:4, 0:7, "/*]")), false)]);
t!(Body, "[f][/*`*/]" => [func!("f", Some((0:4, 0:9, "/*`*/")), true)]);
t!(Body, "[f][/*]" => [func!("f", Some(0:4, 0:7, "/*]"), false)]);
t!(Body, "[f][/*`*/]" => [func!("f", Some(0:4, 0:9, "/*`*/"), true)]);
t!(Body, "[f: //]\n]" => [func!("f: //]\n", None, true)]);
t!(Body, "[f: \"//]\n]" => [func!("f: \"//]\n]", None, false)]);
// End of function with escaped brackets
t!(Body, "[f][\\]]" => [func!("f", Some((0:4, 0:6, "\\]")), true)]);
t!(Body, "[f][\\[]" => [func!("f", Some((0:4, 0:6, "\\[")), true)]);
t!(Body, "[f][\\]]" => [func!("f", Some(0:4, 0:6, "\\]"), true)]);
t!(Body, "[f][\\[]" => [func!("f", Some(0:4, 0:6, "\\["), true)]);
}
#[test]