More highlighting API

benches/oneshot.rs

@@ -6,6 +6,7 @@ use unscanny::Scanner;
 use typst::loading::MemLoader;
 use typst::parse::{parse, TokenMode, Tokens};
 use typst::source::SourceId;
+use typst::syntax::highlight_node;
 use typst::Context;
 
 const SRC: &str = include_str!("bench.typ");
@@ -70,7 +71,13 @@ fn bench_edit(iai: &mut Iai) {
 fn bench_highlight(iai: &mut Iai) {
     let (ctx, id) = context();
     let source = ctx.sources.get(id);
-    iai.run(|| source.highlight(0 .. source.len_bytes(), |_, _| {}));
+    iai.run(|| {
+        highlight_node(
+            source.red().as_ref(),
+            0 .. source.len_bytes(),
+            &mut |_, _| {},
+        )
+    });
 }
 
 fn bench_eval(iai: &mut Iai) {

src/library/text/raw.rs

@@ -1,17 +1,15 @@
-use std::sync::Arc;
-
 use once_cell::sync::Lazy;
 use syntect::easy::HighlightLines;
 use syntect::highlighting::{
-    Color, FontStyle, Highlighter, Style, StyleModifier, Theme, ThemeItem, ThemeSettings,
+    Color, FontStyle, Style, StyleModifier, Theme, ThemeItem, ThemeSettings,
 };
 use syntect::parsing::SyntaxSet;
 
 use super::{FontFamily, Hyphenate, TextNode};
 use crate::library::layout::BlockSpacing;
 use crate::library::prelude::*;
-use crate::source::SourceId;
+use crate::parse::TokenMode;
-use crate::syntax::{self, GreenNode, NodeKind, RedNode};
+use crate::syntax;
 
 /// Monospaced text with optional syntax highlighting.
 #[derive(Debug, Hash)]
@@ -71,20 +69,14 @@ impl Show for RawNode {
             .into();
 
         let mut realized = if matches!(lang.as_deref(), Some("typ" | "typst" | "typc")) {
-            let root = match lang.as_deref() {
+            let mode = match lang.as_deref() {
-                Some("typc") => {
+                Some("typc") => TokenMode::Code,
-                    let children = crate::parse::parse_code(&self.text);
+                _ => TokenMode::Markup,
-                    Arc::new(GreenNode::with_children(NodeKind::CodeBlock, children))
-                }
-                _ => crate::parse::parse(&self.text),
             };
 
-            let red = RedNode::from_root(root, SourceId::from_raw(0));
-            let highlighter = Highlighter::new(&THEME);
-
             let mut seq = vec![];
-            syntax::highlight_syntect(red.as_ref(), &highlighter, &mut |range, style| {
+            syntax::highlight_themed(&self.text, mode, &THEME, &mut |piece, style| {
-                seq.push(styled(&self.text[range], foreground, style));
+                seq.push(styled(piece, foreground, style));
             });
 
             Content::sequence(seq)
@@ -159,12 +151,12 @@ fn styled(piece: &str, foreground: Paint, style: Style) -> Content {
     body
 }
 
-/// The lazily-loaded syntect syntax definitions.
+/// The syntect syntax definitions.
 static SYNTAXES: Lazy<SyntaxSet> = Lazy::new(|| SyntaxSet::load_defaults_newlines());
 
-/// The lazily-loaded theme used for syntax highlighting.
+/// The default theme used for syntax highlighting.
 #[rustfmt::skip]
-static THEME: Lazy<Theme> = Lazy::new(|| Theme {
+pub static THEME: Lazy<Theme> = Lazy::new(|| Theme {
     name: Some("Typst Light".into()),
     author: Some("The Typst Project Developers".into()),
     settings: ThemeSettings::default(),

src/model/property.rs

@@ -131,13 +131,13 @@ impl Debug for KeyId {
 pub trait Key<'a>: Copy + 'static {
     /// The unfolded type which this property is stored as in a style map. For
     /// example, this is [`Toggle`](crate::geom::Length) for the
-    /// [`STRONG`](crate::library::text::TextNode::STRONG) property.
+    /// [`BOLD`](crate::library::text::TextNode::BOLD) property.
     type Value: Debug + Clone + Hash + Sync + Send + 'static;
 
     /// The folded type of value that is returned when reading this property
     /// from a style chain. For example, this is [`bool`] for the
-    /// [`STRONG`](crate::library::text::TextNode::STRONG) property. For
+    /// [`BOLD`](crate::library::text::TextNode::BOLD) property. For non-copy,
-    /// non-copy, non-folding properties this is a reference type.
+    /// non-folding properties this is a reference type.
     type Output;
 
     /// The name of the property, used for debug printing.

src/source.rs

@@ -12,7 +12,7 @@ use crate::diag::TypResult;
 use crate::loading::{FileHash, Loader};
 use crate::parse::{is_newline, parse, reparse};
 use crate::syntax::ast::Markup;
-use crate::syntax::{self, Category, GreenNode, RedNode, Span};
+use crate::syntax::{GreenNode, RedNode, Span};
 use crate::util::{PathExt, StrExt};
 
 #[cfg(feature = "codespan-reporting")]
@@ -269,14 +269,6 @@ impl SourceFile {
         reparse(&mut self.root, &self.src, replace, with.len())
     }
 
-    /// Provide highlighting categories for the given range of the source file.
-    pub fn highlight<F>(&self, range: Range<usize>, mut f: F)
-    where
-        F: FnMut(Range<usize>, Category),
-    {
-        syntax::highlight(self.red().as_ref(), range, &mut f)
-    }
-
     /// Get the length of the file in bytes.
     pub fn len_bytes(&self) -> usize {
         self.src.len()

src/syntax/highlight.rs

@@ -1,13 +1,17 @@
+use std::fmt::Write;
 use std::ops::Range;
+use std::sync::Arc;
 
-use syntect::highlighting::{Highlighter, Style};
+use syntect::highlighting::{Color, FontStyle, Highlighter, Style, Theme};
 use syntect::parsing::Scope;
 
-use super::{NodeKind, RedRef};
+use super::{GreenNode, NodeKind, RedNode, RedRef};
+use crate::parse::TokenMode;
+use crate::source::SourceId;
 
-/// Provide highlighting categories for the children of a node that fall into a
+/// Provide highlighting categories for the descendants of a node that fall into
-/// range.
+/// a range.
-pub fn highlight<F>(node: RedRef, range: Range<usize>, f: &mut F)
+pub fn highlight_node<F>(node: RedRef, range: Range<usize>, f: &mut F)
 where
     F: FnMut(Range<usize>, Category),
 {
@@ -17,30 +21,45 @@ where
             if let Some(category) = Category::determine(child, node, i) {
                 f(span.to_range(), category);
             }
-            highlight(child, range.clone(), f);
+            highlight_node(child, range.clone(), f);
         }
     }
 }
 
-/// Provide syntect highlighting styles for the children of a node.
+/// Highlight source text in a theme by calling `f` with each consecutive piece
-pub fn highlight_syntect<F>(node: RedRef, highlighter: &Highlighter, f: &mut F)
+/// and its style.
+pub fn highlight_themed<F>(text: &str, mode: TokenMode, theme: &Theme, f: &mut F)
 where
-    F: FnMut(Range<usize>, Style),
+    F: FnMut(&str, Style),
 {
-    highlight_syntect_impl(node, vec![], highlighter, f)
+    let root = match mode {
+        TokenMode::Markup => crate::parse::parse(text),
+        TokenMode::Code => {
+            let children = crate::parse::parse_code(text);
+            Arc::new(GreenNode::with_children(NodeKind::CodeBlock, children))
+        }
+    };
+
+    let root = RedNode::from_root(root, SourceId::from_raw(0));
+    let highlighter = Highlighter::new(&theme);
+
+    highlight_themed_impl(text, root.as_ref(), vec![], &highlighter, f);
 }
 
 /// Recursive implementation for returning syntect styles.
-fn highlight_syntect_impl<F>(
+fn highlight_themed_impl<F>(
+    text: &str,
     node: RedRef,
     scopes: Vec<Scope>,
     highlighter: &Highlighter,
     f: &mut F,
 ) where
-    F: FnMut(Range<usize>, Style),
+    F: FnMut(&str, Style),
 {
-    if node.children().size_hint().0 == 0 {
+    if node.children().len() == 0 {
-        f(node.span().to_range(), highlighter.style_for_stack(&scopes));
+        let piece = &text[node.span().to_range()];
+        let style = highlighter.style_for_stack(&scopes);
+        f(piece, style);
         return;
     }
 
@@ -49,10 +68,66 @@ fn highlight_syntect_impl<F>(
         if let Some(category) = Category::determine(child, node, i) {
             scopes.push(Scope::new(category.tm_scope()).unwrap())
         }
-        highlight_syntect_impl(child, scopes, highlighter, f);
+        highlight_themed_impl(text, child, scopes, highlighter, f);
     }
 }
 
+/// Highlight source text into a standalone HTML document.
+pub fn highlight_html(text: &str, mode: TokenMode, theme: &Theme) -> String {
+    let mut buf = String::new();
+    buf.push_str("<!DOCTYPE html>\n");
+    buf.push_str("<html>\n");
+    buf.push_str("<head>\n");
+    buf.push_str("  <meta charset=\"utf-8\">\n");
+    buf.push_str("</head>\n");
+    buf.push_str("<body>\n");
+    buf.push_str(&highlight_pre(text, mode, theme));
+    buf.push_str("\n</body>\n");
+    buf.push_str("</html>\n");
+    buf
+}
+
+/// Highlight source text into an HTML pre element.
+pub fn highlight_pre(text: &str, mode: TokenMode, theme: &Theme) -> String {
+    let mut buf = String::new();
+    buf.push_str("<pre>\n");
+
+    highlight_themed(text, mode, theme, &mut |piece, style| {
+        let styled = style != Style::default();
+        if styled {
+            buf.push_str("<span style=\"");
+
+            if style.foreground != Color::BLACK {
+                let Color { r, g, b, a } = style.foreground;
+                write!(buf, "color: #{r:02x}{g:02x}{b:02x}{a:02x};").unwrap();
+            }
+
+            if style.font_style.contains(FontStyle::BOLD) {
+                buf.push_str("font-weight:bold");
+            }
+
+            if style.font_style.contains(FontStyle::ITALIC) {
+                buf.push_str("font-style:italic");
+            }
+
+            if style.font_style.contains(FontStyle::UNDERLINE) {
+                buf.push_str("text-decoration:underline;")
+            }
+
+            buf.push_str("\">");
+        }
+
+        buf.push_str(piece);
+
+        if styled {
+            buf.push_str("</span>");
+        }
+    });
+
+    buf.push_str("\n</pre>");
+    buf
+}
+
 /// The syntax highlighting category of a node.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
 pub enum Category {
@@ -283,7 +358,8 @@ mod tests {
         fn test(src: &str, goal: &[(Range<usize>, Category)]) {
             let mut vec = vec![];
             let source = SourceFile::detached(src);
-            source.highlight(0 .. src.len(), |range, category| {
+            let full = 0 .. src.len();
+            highlight_node(source.red().as_ref(), full, &mut |range, category| {
                 vec.push((range, category));
             });
             assert_eq!(vec, goal);