Knuth-Plass and MicroType

src/eval/styles.rs

@@ -539,6 +539,11 @@ impl<T> StyleVec<T> {
         self.items.is_empty()
     }
 
+    /// Number of items in the sequence.
+    pub fn len(&self) -> usize {
+        self.items.len()
+    }
+
     /// Iterate over the contained items.
     pub fn items(&self) -> std::slice::Iter<'_, T> {
         self.items.iter()

src/eval/value.rs

@@ -491,6 +491,17 @@ impl<T> Smart<T> {
             Self::Custom(x) => x,
         }
     }
+
+    /// Returns the contained custom value or computes a default value.
+    pub fn unwrap_or_else<F>(self, f: F) -> T
+    where
+        F: FnOnce() -> T,
+    {
+        match self {
+            Self::Auto => f(),
+            Self::Custom(x) => x,
+        }
+    }
 }
 
 impl<T> Default for Smart<T> {

src/geom/em.rs

@@ -108,3 +108,9 @@ assign_impl!(Em += Em);
 assign_impl!(Em -= Em);
 assign_impl!(Em *= f64);
 assign_impl!(Em /= f64);
+
+impl Sum for Em {
+    fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
+        Self(iter.map(|s| s.0).sum())
+    }
+}

src/geom/relative.rs

@@ -43,6 +43,11 @@ impl Relative {
         self.0 == 0.0
     }
 
+    /// Whether the ratio is one.
+    pub fn is_one(self) -> bool {
+        self.0 == 1.0
+    }
+
     /// The absolute value of the this relative.
     pub fn abs(self) -> Self {
         Self::new(self.get().abs())

src/library/text/mod.rs

@@ -56,6 +56,10 @@ impl TextNode {
     pub const SIZE: Linear = Length::pt(11.0).into();
     /// The amount of space that should be added between characters.
     pub const TRACKING: Em = Em::zero();
+    /// The ratio by which spaces should be stretched.
+    pub const SPACING: Relative = Relative::one();
+    /// Whether glyphs can hang over into the margin.
+    pub const OVERHANG: bool = true;
     /// The top end of the text bounding box.
     pub const TOP_EDGE: VerticalFontMetric = VerticalFontMetric::CapHeight;
     /// The bottom end of the text bounding box.

src/library/text/par.rs

@@ -35,6 +35,8 @@ impl ParNode {
     pub const ALIGN: Align = Align::Left;
     /// Whether to justify text in its line.
     pub const JUSTIFY: bool = false;
+    /// How to determine line breaks.
+    pub const LINEBREAKS: Smart<Linebreaks> = Smart::Auto;
     /// Whether to hyphenate text to improve line breaking. When `auto`, words
     /// will will be hyphenated if and only if justification is enabled.
     pub const HYPHENATE: Smart<bool> = Smart::Auto;
@@ -85,6 +87,7 @@ impl ParNode {
         styles.set_opt(Self::DIR, dir);
         styles.set_opt(Self::ALIGN, align);
         styles.set_opt(Self::JUSTIFY, args.named("justify")?);
+        styles.set_opt(Self::LINEBREAKS, args.named("linebreaks")?);
         styles.set_opt(Self::HYPHENATE, args.named("hyphenate")?);
         styles.set_opt(Self::LEADING, args.named("leading")?);
         styles.set_opt(Self::SPACING, args.named("spacing")?);
@@ -176,6 +179,25 @@ impl Merge for ParChild {
     }
 }
 
+/// How to determine line breaks in a paragraph.
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
+pub enum Linebreaks {
+    /// Determine the linebreaks in a simple first-fit style.
+    Simple,
+    /// Optimize the linebreaks for the whole paragraph.
+    Optimized,
+}
+
+castable! {
+    Linebreaks,
+    Expected: "string",
+    Value::Str(string) => match string.as_str() {
+        "simple" => Self::Simple,
+        "optimized" => Self::Optimized,
+        _ => Err(r#"expected "simple" or "optimized""#)?,
+    },
+}
+
 /// A paragraph break.
 pub struct ParbreakNode;
 
@@ -266,6 +288,9 @@ struct Line<'a> {
     fr: Fractional,
     /// Whether the line ends at a mandatory break.
     mandatory: bool,
+    /// Whether the line ends with a hyphen or dash, either naturally or through
+    /// hyphenation.
+    dash: bool,
 }
 
 impl<'a> Line<'a> {
@@ -283,6 +308,28 @@ impl<'a> Line<'a> {
     fn get(&self, index: usize) -> Option<&ParItem<'a>> {
         self.items().nth(index)
     }
+
+    // How many spaces the line contains.
+    fn spaces(&self) -> usize {
+        let mut spaces = 0;
+        for item in self.items() {
+            if let ParItem::Text(shaped) = item {
+                spaces += shaped.spaces();
+            }
+        }
+        spaces
+    }
+
+    /// How much of the line is stretchable spaces.
+    fn stretch(&self) -> Length {
+        let mut stretch = Length::zero();
+        for item in self.items() {
+            if let ParItem::Text(shaped) = item {
+                stretch += shaped.stretch();
+            }
+        }
+        stretch
+    }
 }
 
 /// Prepare paragraph layout by shaping the whole paragraph and layouting all
@@ -323,7 +370,7 @@ fn prepare<'a>(
             }
             ParChild::Spacing(spacing) => match *spacing {
                 Spacing::Linear(v) => {
-                    let resolved = v.resolve(regions.first.x);
+                    let resolved = v.resolve(regions.base.x);
                     items.push(ParItem::Absolute(resolved));
                     ranges.push(range);
                 }
@@ -333,6 +380,16 @@ fn prepare<'a>(
                 }
             },
             ParChild::Node(node) => {
+                // Prevent margin overhang in the inline node except if there's
+                // just this one.
+                let local;
+                let styles = if par.0.len() != 1 {
+                    local = StyleMap::with(TextNode::OVERHANG, false);
+                    local.chain(&styles)
+                } else {
+                    styles
+                };
+
                 let size = Size::new(regions.first.x, regions.base.y);
                 let pod = Regions::one(size, regions.base, Spec::splat(false));
                 let frame = node.layout(ctx, &pod, styles)?.remove(0);
@@ -345,19 +402,42 @@ fn prepare<'a>(
     Ok(Preparation { bidi, items, ranges })
 }
 
-/// Perform line breaking.
+/// Find suitable linebreaks.
 fn linebreak<'a>(
     p: &'a Preparation<'a>,
     fonts: &mut FontStore,
     width: Length,
     styles: StyleChain,
 ) -> Vec<Line<'a>> {
-    // The already determined lines and the current line attempt.
+    let breaks = styles.get(ParNode::LINEBREAKS).unwrap_or_else(|| {
+        if styles.get(ParNode::JUSTIFY) {
+            Linebreaks::Optimized
+        } else {
+            Linebreaks::Simple
+        }
+    });
+
+    let breaker = match breaks {
+        Linebreaks::Simple => linebreak_simple,
+        Linebreaks::Optimized => linebreak_optimized,
+    };
+
+    breaker(p, fonts, width, styles)
+}
+
+/// Perform line breaking in simple first-fit style. This means that we build
+/// lines a greedily, always taking the longest possible line. This may lead to
+/// very unbalanced line, but is fast and simple.
+fn linebreak_simple<'a>(
+    p: &'a Preparation<'a>,
+    fonts: &mut FontStore,
+    width: Length,
+    styles: StyleChain,
+) -> Vec<Line<'a>> {
     let mut lines = vec![];
     let mut start = 0;
     let mut last = None;
 
-    // Find suitable line breaks.
     for (end, mandatory, hyphen) in breakpoints(&p.bidi.text, styles) {
         // Compute the line and its size.
         let mut attempt = line(p, fonts, start .. end, mandatory, hyphen);
@@ -392,6 +472,133 @@ fn linebreak<'a>(
     lines
 }
 
+/// Perform line breaking in optimized Knuth-Plass style. Here, we use more
+/// context to determine the line breaks than in the simple first-fit style. For
+/// example, we might choose to cut a line short even though there is still a
+/// bit of space to improve the fit of one of the following lines. The
+/// Knuth-Plass algorithm is based on the idea of "cost". A line which has a
+/// very tight or very loose fit has a higher cost than one that is just right.
+/// Ending a line with a hyphen incurs extra cost and endings two successive
+/// lines with hyphens even more.
+///
+/// To find the layout with the minimal total cost the algorithm uses dynamic
+/// programming: For each possible breakpoint it determines the optimal
+/// paragraph layout _up to that point_. It walks over all possible start points
+/// for a line ending at that point and finds the one for which the cost of the
+/// line plus the cost of the optimal paragraph up to the start point (already
+/// computed and stored in dynamic programming table) is minimal. The final
+/// result is simply the layout determined for the last breakpoint at the end of
+/// text.
+fn linebreak_optimized<'a>(
+    p: &'a Preparation<'a>,
+    fonts: &mut FontStore,
+    width: Length,
+    styles: StyleChain,
+) -> Vec<Line<'a>> {
+    /// The cost of a line or paragraph layout.
+    type Cost = f64;
+
+    /// An entry in the dynamic programming table.
+    struct Entry<'a> {
+        pred: usize,
+        total: Cost,
+        line: Line<'a>,
+    }
+
+    // Cost parameters.
+    const HYPH_COST: Cost = 0.5;
+    const CONSECUTIVE_DASH_COST: Cost = 30.0;
+    const MAX_COST: Cost = 10_000.0;
+    const MIN_COST: Cost = -MAX_COST;
+    const MIN_RATIO: f64 = -0.15;
+
+    // Density parameters.
+    let justify = styles.get(ParNode::JUSTIFY);
+
+    // Dynamic programming table.
+    let mut active = 0;
+    let mut table = vec![Entry {
+        pred: 0,
+        total: 0.0,
+        line: line(p, fonts, 0 .. 0, false, false),
+    }];
+
+    for (end, mandatory, hyphen) in breakpoints(&p.bidi.text, styles) {
+        let k = table.len();
+        let eof = end == p.bidi.text.len();
+        let mut best: Option<Entry> = None;
+
+        // Find the optimal predecessor.
+        for (i, pred) in table.iter_mut().enumerate().skip(active) {
+            // Layout the line.
+            let start = pred.line.range.end;
+            let attempt = line(p, fonts, start .. end, mandatory, hyphen);
+
+            // Determine how much the line's spaces would need to be stretched
+            // to make it the desired width.
+            let mut ratio = (width - attempt.size.x) / attempt.stretch();
+            if ratio.is_infinite() {
+                ratio = ratio.signum() * MAX_COST;
+            }
+
+            // Determine the cost of the line.
+            let mut cost = if ratio < if justify { MIN_RATIO } else { 0.0 } {
+                // The line is overfull. This is the case if
+                // - justification is on, but we'd need to shrink to much
+                // - justification is off and the line just doesn't fit
+                // Since any longer line will also be overfull, we can deactive
+                // this breakpoint.
+                active = i + 1;
+                MAX_COST
+            } else if eof {
+                // This is the final line and its not overfull since then
+                // we would have taken the above branch.
+                0.0
+            } else if mandatory {
+                // This is a mandatory break and the line is not overfull, so it
+                // has minimum cost. All breakpoints before this one become
+                // inactive since no line can span above the mandatory break.
+                active = k;
+                MIN_COST
+            } else {
+                // Normal line with cost of |ratio^3|.
+                ratio.powi(3).abs()
+            };
+
+            // Penalize hyphens and especially two consecutive hyphens.
+            if hyphen {
+                cost += HYPH_COST;
+            }
+            if attempt.dash && pred.line.dash {
+                cost += CONSECUTIVE_DASH_COST;
+            }
+
+            // The total cost of this line and its chain of predecessors.
+            let total = pred.total + cost;
+
+            // If this attempt is better than what we had before, take it!
+            if best.as_ref().map_or(true, |best| best.total >= total) {
+                best = Some(Entry { pred: i, total, line: attempt });
+            }
+        }
+
+        table.push(best.unwrap());
+    }
+
+    // Retrace the best path.
+    let mut lines = vec![];
+    let mut idx = table.len() - 1;
+    while idx != 0 {
+        table.truncate(idx + 1);
+        let entry = table.pop().unwrap();
+        lines.push(entry.line);
+        idx = entry.pred;
+    }
+
+    lines.reverse();
+    lines
+}
+
 /// Determine all possible points in the text where lines can broken.
 ///
 /// Returns for each breakpoint the text index, whether the break is mandatory
@@ -415,6 +622,10 @@ fn breakpoints<'a>(
 
     if let Some(lang) = lang {
         Either::Left(breaks.flat_map(move |(end, mandatory)| {
+            // We don't want to confuse the hyphenator with trailing
+            // punctuation, so we trim it. And if that makes the word empty, we
+            // need to return the single breakpoint manually because hypher
+            // would eat it.
             let word = &text[last .. end];
             let trimmed = word.trim_end_matches(|c: char| !c.is_alphabetic());
             let suffix = last + trimmed.len();
@@ -441,7 +652,7 @@ fn breakpoints<'a>(
 fn line<'a>(
     p: &'a Preparation,
     fonts: &mut FontStore,
-    mut range: Range,
+    range: Range,
     mandatory: bool,
     hyphen: bool,
 ) -> Line<'a> {
@@ -453,56 +664,62 @@ fn line<'a>(
         p.find(range.start).unwrap()
     };
 
-    // Slice out the relevant items and ranges.
+    // Slice out the relevant items.
     let mut items = &p.items[first_idx ..= last_idx];
-    let ranges = &p.ranges[first_idx ..= last_idx];
 
     // Reshape the last item if it's split in half.
     let mut last = None;
-    if let Some((ParItem::Text(shaped), rest)) = items.split_last() {
+    let mut dash = false;
+    if let Some((ParItem::Text(shaped), before)) = items.split_last() {
         // Compute the range we want to shape, trimming whitespace at the
         // end of the line.
         let base = p.ranges[last_idx].start;
         let start = range.start.max(base);
-        let end = start + p.bidi.text[start .. range.end].trim_end().len();
-        let shifted = start - base .. end - base;
+        let trimmed = p.bidi.text[start .. range.end].trim_end();
+        let shy = trimmed.ends_with('\u{ad}');
+        dash = hyphen || shy || trimmed.ends_with(['-', '–', '—']);
 
-        // Reshape if necessary.
-        if shifted.len() < shaped.text.len() {
-            // If start == end and the rest is empty, then we have an empty
-            // line. To make that line have the appropriate height, we shape the
-            // empty string.
-            if !shifted.is_empty() || rest.is_empty() {
-                // Reshape that part.
+        // Usually, we don't want to shape an empty string because:
+        // - We don't want the height of trimmed whitespace in a different
+        //   font to be considered for the line height.
+        // - Even if it's in the same font, its unnecessary.
+        //
+        // There is one exception though. When the whole line is empty, we
+        // need the shaped empty string to make the line the appropriate
+        // height. That is the case exactly if the string is empty and there
+        // are no other items in the line.
+        if hyphen || trimmed.len() < shaped.text.len() {
+            if hyphen || !trimmed.is_empty() || before.is_empty() {
+                let end = start + trimmed.len();
+                let shifted = start - base .. end - base;
                 let mut reshaped = shaped.reshape(fonts, shifted);
-                if hyphen {
+                if hyphen || shy {
                     reshaped.push_hyphen(fonts);
                 }
                 last = Some(ParItem::Text(reshaped));
             }
 
-            items = rest;
-            range.end = end;
+            items = before;
         }
     }
 
     // Reshape the start item if it's split in half.
     let mut first = None;
-    if let Some((ParItem::Text(shaped), rest)) = items.split_first() {
+    if let Some((ParItem::Text(shaped), after)) = items.split_first() {
         // Compute the range we want to shape.
         let Range { start: base, end: first_end } = p.ranges[first_idx];
         let start = range.start;
         let end = range.end.min(first_end);
-        let shifted = start - base .. end - base;
 
         // Reshape if necessary.
-        if shifted.len() < shaped.text.len() {
-            if !shifted.is_empty() {
+        if end - start < shaped.text.len() {
+            if start < end {
+                let shifted = start - base .. end - base;
                 let reshaped = shaped.reshape(fonts, shifted);
                 first = Some(ParItem::Text(reshaped));
             }
 
-            items = rest;
+            items = after;
         }
     }
 
@@ -535,11 +752,12 @@ fn line<'a>(
         first,
         items,
         last,
-        ranges,
+        ranges: &p.ranges[first_idx ..= last_idx],
         size: Size::new(width, top + bottom),
         baseline: top,
         fr,
         mandatory,
+        dash,
     }
 }
 
@@ -603,30 +821,59 @@ fn commit(
     justify: bool,
 ) -> Frame {
     let size = Size::new(width, line.size.y);
-
     let mut remaining = width - line.size.x;
     let mut offset = Length::zero();
+
+    // Reorder the line from logical to visual order.
+    let reordered = reorder(line);
+
+    // Handle hanging punctuation to the left.
+    if let Some(ParItem::Text(text)) = reordered.first() {
+        if let Some(glyph) = text.glyphs.first() {
+            if text.styles.get(TextNode::OVERHANG) {
+                let start = text.dir.is_positive();
+                let em = text.styles.get(TextNode::SIZE).abs;
+                let amount = overhang(glyph.c, start) * glyph.x_advance.resolve(em);
+                offset -= amount;
+                remaining += amount;
+            }
+        }
+    }
+
+    // Handle hanging punctuation to the right.
+    if let Some(ParItem::Text(text)) = reordered.last() {
+        if let Some(glyph) = text.glyphs.last() {
+            if text.styles.get(TextNode::OVERHANG)
+                && (reordered.len() > 1 || text.glyphs.len() > 1)
+            {
+                let start = !text.dir.is_positive();
+                let em = text.styles.get(TextNode::SIZE).abs;
+                let amount = overhang(glyph.c, start) * glyph.x_advance.resolve(em);
+                remaining += amount;
+            }
+        }
+    }
+
+    // Determine how much to justify each space.
+    let mut justification = Length::zero();
+    if remaining < Length::zero()
+        || (justify
+            && !line.mandatory
+            && line.range.end < line.bidi.text.len()
+            && line.fr.is_zero())
+    {
+        let spaces = line.spaces();
+        if spaces > 0 {
+            justification = remaining / spaces as f64;
+            remaining = Length::zero();
+        }
+    }
+
     let mut output = Frame::new(size);
     output.baseline = Some(line.baseline);
 
-    let mut justification = Length::zero();
-    if justify
-        && !line.mandatory
-        && line.range.end < line.bidi.text.len()
-        && line.fr.is_zero()
-    {
-        let mut spaces = 0;
-        for item in line.items() {
-            if let ParItem::Text(shaped) = item {
-                spaces += shaped.spaces();
-            }
-        }
-
-        justification = remaining / spaces as f64;
-        remaining = Length::zero();
-    }
-
-    for item in reorder(line) {
+    // Construct the line's frame from left to right.
+    for item in reordered {
         let mut position = |frame: Frame| {
             let x = offset + align.resolve(remaining);
             let y = line.baseline - frame.baseline();
@@ -645,37 +892,67 @@ fn commit(
     output
 }
 
-/// Iterate through a line's items in visual order.
-fn reorder<'a>(line: &'a Line<'a>) -> impl Iterator<Item = &'a ParItem<'a>> {
+/// Return a line's items in visual order.
+fn reorder<'a>(line: &'a Line<'a>) -> Vec<&'a ParItem<'a>> {
+    let mut reordered = vec![];
+
     // The bidi crate doesn't like empty lines.
-    let (levels, runs) = if !line.range.is_empty() {
-        // Find the paragraph that contains the line.
-        let para = line
-            .bidi
-            .paragraphs
-            .iter()
-            .find(|para| para.range.contains(&line.range.start))
-            .unwrap();
+    if line.range.is_empty() {
+        return reordered;
+    }
 
-        // Compute the reordered ranges in visual order (left to right).
-        line.bidi.visual_runs(para, line.range.clone())
-    } else {
-        (vec![], vec![])
-    };
+    // Find the paragraph that contains the line.
+    let para = line
+        .bidi
+        .paragraphs
+        .iter()
+        .find(|para| para.range.contains(&line.range.start))
+        .unwrap();
 
-    runs.into_iter()
-        .flat_map(move |run| {
-            let first_idx = line.find(run.start).unwrap();
-            let last_idx = line.find(run.end - 1).unwrap();
-            let range = first_idx ..= last_idx;
+    // Compute the reordered ranges in visual order (left to right).
+    let (levels, runs) = line.bidi.visual_runs(para, line.range.clone());
 
-            // Provide the items forwards or backwards depending on the run's
-            // direction.
-            if levels[run.start].is_ltr() {
-                Either::Left(range)
-            } else {
-                Either::Right(range.rev())
-            }
-        })
-        .map(move |idx| line.get(idx).unwrap())
+    // Collect the reordered items.
+    for run in runs {
+        let first_idx = line.find(run.start).unwrap();
+        let last_idx = line.find(run.end - 1).unwrap();
+        let range = first_idx ..= last_idx;
+
+        // Provide the items forwards or backwards depending on the run's
+        // direction.
+        if levels[run.start].is_ltr() {
+            reordered.extend(range.filter_map(|i| line.get(i)));
+        } else {
+            reordered.extend(range.rev().filter_map(|i| line.get(i)));
+        }
+    }
+
+    reordered
+}
+
+/// How much a character should hang into the margin.
+///
+/// For selection of overhang characters, see also:
+/// https://recoveringphysicist.com/21/
+fn overhang(c: char, start: bool) -> f64 {
+    match c {
+        '“' | '”' | '„' | '‟' | '"' if start => 1.0,
+        '‘' | '’' | '‚' | '‛' | '\'' if start => 1.0,
+
+        '“' | '”' | '„' | '‟' | '"' if !start => 0.6,
+        '‘' | '’' | '‚' | '‛' | '\'' if !start => 0.6,
+        '–' | '—' if !start => 0.2,
+        '-' if !start => 0.55,
+
+        '.' | ',' => 0.8,
+        ':' | ';' => 0.3,
+        '«' | '»' => 0.2,
+        '‹' | '›' => 0.4,
+
+        // Arabic and Ideographic
+        '\u{60C}' | '\u{6D4}' => 0.4,
+        '\u{3001}' | '\u{3002}' => 1.0,
+
+        _ => 0.0,
+    }
 }

src/library/text/shaping.rs

@@ -39,14 +39,21 @@ pub struct ShapedGlyph {
     pub x_advance: Em,
     /// The horizontal offset of the glyph.
     pub x_offset: Em,
-    /// The start index of the glyph in the source text.
-    pub text_index: usize,
+    /// A value that is the same for all glyphs belong to one cluster.
+    pub cluster: usize,
     /// Whether splitting the shaping result before this glyph would yield the
     /// same results as shaping the parts to both sides of `text_index`
     /// separately.
     pub safe_to_break: bool,
-    /// Whether this glyph represents a space.
-    pub is_space: bool,
+    /// The first char in this glyph's cluster.
+    pub c: char,
+}
+
+impl ShapedGlyph {
+    /// Whether the glyph is a justifiable space.
+    pub fn is_space(&self) -> bool {
+        self.c == ' '
+    }
 }
 
 /// A side you can go toward.
@@ -77,7 +84,7 @@ impl<'a> ShapedText<'a> {
                 .map(|glyph| Glyph {
                     id: glyph.glyph_id,
                     x_advance: glyph.x_advance
-                        + if glyph.is_space {
+                        + if glyph.is_space() {
                             frame.size.x += justification;
                             Em::from_length(justification, size)
                         } else {
@@ -110,7 +117,17 @@ impl<'a> ShapedText<'a> {
 
     /// How many spaces the text contains.
     pub fn spaces(&self) -> usize {
-        self.glyphs.iter().filter(|g| g.is_space).count()
+        self.glyphs.iter().filter(|g| g.is_space()).count()
+    }
+
+    /// The width of the spaces in the text.
+    pub fn stretch(&self) -> Length {
+        self.glyphs
+            .iter()
+            .filter(|g| g.is_space())
+            .map(|g| g.x_advance)
+            .sum::<Em>()
+            .resolve(self.styles.get(TextNode::SIZE).abs)
     }
 
     /// Reshape a range of the shaped text, reusing information from this
@@ -121,43 +138,40 @@ impl<'a> ShapedText<'a> {
         text_range: Range<usize>,
     ) -> ShapedText<'a> {
         if let Some(glyphs) = self.slice_safe_to_break(text_range.clone()) {
-            let (size, baseline) = measure(fonts, glyphs, self.styles);
+            let (size, baseline) = measure(fonts, &glyphs, self.styles);
             Self {
                 text: Cow::Borrowed(&self.text[text_range]),
                 dir: self.dir,
-                styles: self.styles.clone(),
+                styles: self.styles,
                 size,
                 baseline,
                 glyphs: Cow::Borrowed(glyphs),
             }
         } else {
-            shape(fonts, &self.text[text_range], self.styles.clone(), self.dir)
+            shape(fonts, &self.text[text_range], self.styles, self.dir)
         }
     }
 
     /// Push a hyphen to end of the text.
     pub fn push_hyphen(&mut self, fonts: &mut FontStore) {
-        // When there are no glyphs, we just use the vertical metrics of the
-        // first available font.
         let size = self.styles.get(TextNode::SIZE).abs;
         let variant = variant(self.styles);
         families(self.styles).find_map(|family| {
-            // Allow hyphens to overhang a bit.
-            const INSET: f64 = 0.4;
             let face_id = fonts.select(family, variant)?;
             let face = fonts.get(face_id);
             let ttf = face.ttf();
             let glyph_id = ttf.glyph_index('-')?;
             let x_advance = face.to_em(ttf.glyph_hor_advance(glyph_id)?);
-            self.size.x += INSET * x_advance.resolve(size);
+            let cluster = self.glyphs.last().map(|g| g.cluster).unwrap_or_default();
+            self.size.x += x_advance.resolve(size);
             self.glyphs.to_mut().push(ShapedGlyph {
                 face_id,
                 glyph_id: glyph_id.0,
                 x_advance,
                 x_offset: Em::zero(),
-                text_index: self.text.len(),
+                cluster,
                 safe_to_break: true,
-                is_space: false,
+                c: '-',
             });
             Some(())
         });
@@ -193,7 +207,7 @@ impl<'a> ShapedText<'a> {
         let mut idx = self
             .glyphs
             .binary_search_by(|g| {
-                let ordering = g.text_index.cmp(&text_index);
+                let ordering = g.cluster.cmp(&text_index);
                 if ltr { ordering } else { ordering.reverse() }
             })
             .ok()?;
@@ -205,7 +219,7 @@ impl<'a> ShapedText<'a> {
 
         // Search for the outermost glyph with the text index.
         while let Some(next) = next(idx, 1) {
-            if self.glyphs.get(next).map_or(true, |g| g.text_index != text_index) {
+            if self.glyphs.get(next).map_or(true, |g| g.cluster != text_index) {
                 break;
             }
             idx = next;
@@ -249,7 +263,12 @@ pub fn shape<'a>(
         );
     }
 
-    track(&mut glyphs, styles.get(TextNode::TRACKING));
+    track_and_space(
+        &mut glyphs,
+        styles.get(TextNode::TRACKING),
+        styles.get(TextNode::SPACING),
+    );
+
     let (size, baseline) = measure(fonts, &glyphs, styles);
 
     ShapedText {
@@ -456,9 +475,9 @@ fn shape_segment<'a>(
                 glyph_id: info.glyph_id as u16,
                 x_advance: face.to_em(pos[i].x_advance),
                 x_offset: face.to_em(pos[i].x_offset),
-                text_index: base + cluster,
+                cluster: base + cluster,
                 safe_to_break: !info.unsafe_to_break(),
-                is_space: text[cluster ..].chars().next() == Some(' '),
+                c: text[cluster ..].chars().next().unwrap(),
             });
         } else {
             // Determine the source text range for the tofu sequence.
@@ -518,18 +537,19 @@ fn shape_segment<'a>(
     }
 }
 
-/// Apply tracking to a slice of shaped glyphs.
-fn track(glyphs: &mut [ShapedGlyph], tracking: Em) {
-    if tracking.is_zero() {
+/// Apply tracking and spacing to a slice of shaped glyphs.
+fn track_and_space(glyphs: &mut [ShapedGlyph], tracking: Em, spacing: Relative) {
+    if tracking.is_zero() && spacing.is_one() {
         return;
     }
 
     let mut glyphs = glyphs.iter_mut().peekable();
     while let Some(glyph) = glyphs.next() {
-        if glyphs
-            .peek()
-            .map_or(false, |next| glyph.text_index != next.text_index)
-        {
+        if glyph.is_space() {
+            glyph.x_advance *= spacing.get();
+        }
+
+        if glyphs.peek().map_or(false, |next| glyph.cluster != next.cluster) {
             glyph.x_advance += tracking;
         }
     }

src/loading/fs.rs

@@ -154,6 +154,8 @@ mod tests {
         paths.sort();
 
         assert_eq!(paths, [
+            Path::new("fonts/CMU-Serif-Bold.ttf"),
+            Path::new("fonts/CMU-Serif-Regular.ttf"),
             Path::new("fonts/IBMPlexMono-Regular.ttf"),
             Path::new("fonts/IBMPlexSans-Bold.ttf"),
             Path::new("fonts/IBMPlexSans-BoldItalic.ttf"),

tests/typ/code/closure.typ

@@ -8,6 +8,7 @@
 #let x = "\"hi\""
 
 // Should output `"hi" => "bye"`.
+#set text(overhang: false)
 #x => "bye"
 
 ---

tests/typ/text/hyphenate.typ

@@ -1,14 +1,30 @@
 // Test hyphenation.
 
 ---
+// Hyphenate english.
 #set page(width: 70pt)
 #set par(lang: "en", hyphenate: true)
 Warm welcomes to Typst.
 
-#h(6pt) networks, the rest.
-
 ---
+// Hyphenate greek.
 #set page(width: 60pt)
 #set par(lang: "el", hyphenate: true)
 διαμερίσματα. \
 λατρευτός
+
+---
+// Hyphenate between shape runs.
+#set par(lang: "en", hyphenate: true)
+#set page(width: 80pt)
+
+It's a #emph[Tree]beard.
+
+---
+// This sequence would confuse hypher if we passed trailing / leading
+// punctuation instead of just the words. So this tests that we don't
+// do that. The test passes if there's just one hyphenation between
+// "net" and "works".
+#set page(width: 70pt)
+#set par(lang: "en", hyphenate: true)
+#h(6pt) networks, the rest.

tests/typ/text/knuth.typ

@@ -0,0 +1,29 @@
+#set page(width: auto, height: auto)
+#set par(lang: "en", leading: 3pt, justify: true)
+#set text(family: "CMU Serif")
+
+#let story = [
+  In olden times when wishing still helped one, there lived a king whose
+  daughters were all beautiful; and the youngest was so beautiful that the sun
+  itself, which has seen so much, was astonished whenever it shone in her face.
+  Close by the king’s castle lay a great dark forest, and under an old lime-tree
+  in the forest was a well, and when the day was very warm, the king’s child
+  went out into the forest and sat down by the side of the cool fountain; and
+  when she was bored she took a golden ball, and threw it up on high and caught
+  it; and this ball was her favorite plaything.
+]
+
+#let column(title, linebreaks, hyphenate) = {
+  rect(width: 132pt, fill: rgb("eee"))[
+    #strong(title)
+    #par(linebreaks: linebreaks, hyphenate: hyphenate, story)
+  ]
+}
+
+#grid(
+  columns: 3,
+  gutter: 10pt,
+  column([Simple without hyphens], "simple", false),
+  column([Simple with hyphens], "simple", true),
+  column([Optimized with hyphens], "optimized", true),
+)

tests/typ/text/microtype.typ

@@ -0,0 +1,35 @@
+// Test micro-typographical shenanigans.
+
+---
+// Test that overhang is off by default in boxes.
+A#box["]B
+
+---
+// Test justified quotes.
+#set par(justify: true)
+“A quote that hangs a bit into the margin.” \
+  --- somebody
+
+---
+// Test fancy quotes in the left margin.
+#set par(align: right)
+»Book quotes are even smarter.« \
+›Book quotes are even smarter.‹ \
+
+---
+// Test fancy quotes in the right margin.
+#set par(align: left)
+«Book quotes are even smarter.» \
+‹Book quotes are even smarter.› \
+
+---
+#set par(lang: "ar")
+#set text("Noto Sans Arabic")
+"المطر هو الحياة" \
+المطر هو الحياة
+
+---
+// Test that lone punctuation doesn't overhang into the margin.
+#set page(margins: 0pt)
+#set par(align: right)
+: