typst/docs/src/lib.rs

//! Documentation provider for Typst.

mod html;

pub use html::Html;

use std::fmt::{self, Debug, Formatter};
use std::path::Path;

use comemo::Prehashed;
use heck::ToTitleCase;
use include_dir::{include_dir, Dir};
use once_cell::sync::Lazy;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_yaml as yaml;
use typst::doc::Frame;
use typst::font::{Font, FontBook};
use typst::geom::{Abs, Sides, Smart};
use typst::model::{CastInfo, Func, FuncInfo, Library, Module, ParamInfo, Value};
use typst_library::layout::PageNode;
use unscanny::Scanner;

static SRC: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/src");
static FILES: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../assets/files");
static IMAGES: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../assets/images");
static DETAILS: Lazy<yaml::Mapping> = Lazy::new(|| yaml("reference/details.yml"));
static GROUPS: Lazy<Vec<GroupData>> = Lazy::new(|| yaml("reference/groups.yml"));

static FONTS: Lazy<(Prehashed<FontBook>, Vec<Font>)> = Lazy::new(|| {
    static DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/../assets/fonts");
    let fonts: Vec<_> = DIR
        .files()
        .flat_map(|file| Font::iter(file.contents().into()))
        .collect();
    let book = FontBook::from_fonts(&fonts);
    (Prehashed::new(book), fonts)
});

static LIBRARY: Lazy<Prehashed<Library>> = Lazy::new(|| {
    let mut lib = typst_library::build();
    lib.styles.set(PageNode::WIDTH, Smart::Custom(Abs::pt(240.0).into()));
    lib.styles.set(PageNode::HEIGHT, Smart::Auto);
    lib.styles
        .set(PageNode::MARGIN, Sides::splat(Some(Smart::Custom(Abs::pt(15.0).into()))));
    typst::model::set_lang_items(lib.items.clone());
    Prehashed::new(lib)
});

/// Build documentation pages.
pub fn provide(resolver: &dyn Resolver) -> Vec<PageModel> {
    vec![
        markdown_page(resolver, "/docs/", "general/overview.md").with_route("/docs/"),
        tutorial_page(resolver),
        reference_page(resolver),
        markdown_page(resolver, "/docs/", "general/changelog.md"),
        markdown_page(resolver, "/docs/", "general/community.md"),
    ]
}

/// Resolve consumer dependencies.
pub trait Resolver {
    /// Try to resolve a link that the system cannot resolve itself.
    fn link(&self, link: &str) -> Option<String>;

    /// Produce an URL for an image file.
    fn image(&self, filename: &str, data: &[u8]) -> String;

    /// Produce HTML for an example.
    fn example(&self, source: Html, frames: &[Frame]) -> Html;
}

/// Details about a documentation page and its children.
#[derive(Debug, Serialize)]
pub struct PageModel {
    pub route: String,
    pub title: String,
    pub description: String,
    pub part: Option<&'static str>,
    pub body: BodyModel,
    pub children: Vec<Self>,
}

impl PageModel {
    fn with_route(self, route: &str) -> Self {
        Self { route: route.into(), ..self }
    }

    fn with_part(self, part: &'static str) -> Self {
        Self { part: Some(part), ..self }
    }
}

/// Details about the body of a documentation page.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
#[serde(tag = "kind", content = "content")]
pub enum BodyModel {
    Html(Html),
    Category(CategoryModel),
    Func(FuncModel),
    Funcs(FuncsModel),
    Type(TypeModel),
    Symbols(SymbolsModel),
}

/// Build the tutorial.
fn tutorial_page(resolver: &dyn Resolver) -> PageModel {
    let mut page = markdown_page(resolver, "/docs/", "tutorial/welcome.md");
    page.children = SRC
        .get_dir("tutorial")
        .unwrap()
        .files()
        .filter(|file| file.path() != Path::new("tutorial/welcome.md"))
        .map(|file| markdown_page(resolver, "/docs/tutorial/", file.path()))
        .collect();
    page
}

/// Build the reference.
fn reference_page(resolver: &dyn Resolver) -> PageModel {
    let mut page = markdown_page(resolver, "/docs/", "reference/welcome.md");
    page.children = vec![
        markdown_page(resolver, "/docs/reference/", "reference/syntax.md")
            .with_part("Language"),
        markdown_page(resolver, "/docs/reference/", "reference/styling.md"),
        markdown_page(resolver, "/docs/reference/", "reference/scripting.md"),
        types_page(resolver, "/docs/reference/"),
        category_page(resolver, "text").with_part("Content"),
        category_page(resolver, "math"),
        category_page(resolver, "layout"),
        category_page(resolver, "visualize"),
        category_page(resolver, "meta"),
        category_page(resolver, "symbols"),
        category_page(resolver, "foundations").with_part("Compute"),
        category_page(resolver, "calculate"),
        category_page(resolver, "construct"),
        category_page(resolver, "data-loading"),
    ];
    page
}

/// Create a page from a markdown file.
#[track_caller]
fn markdown_page(
    resolver: &dyn Resolver,
    parent: &str,
    path: impl AsRef<Path>,
) -> PageModel {
    assert!(parent.starts_with('/') && parent.ends_with('/'));
    let md = SRC.get_file(path).unwrap().contents_utf8().unwrap();
    let html = Html::markdown(resolver, md);
    let title = html.title().expect("chapter lacks a title").to_string();
    PageModel {
        route: format!("{parent}{}/", urlify(&title)),
        title,
        description: html.description().unwrap(),
        part: None,
        body: BodyModel::Html(html),
        children: vec![],
    }
}

/// Details about a category.
#[derive(Debug, Serialize)]
pub struct CategoryModel {
    pub name: String,
    pub details: Html,
    pub kind: &'static str,
    pub items: Vec<CategoryItem>,
}

/// Details about a category item.
#[derive(Debug, Serialize)]
pub struct CategoryItem {
    pub name: String,
    pub route: String,
    pub oneliner: String,
    pub code: bool,
}

/// Create a page for a category.
#[track_caller]
fn category_page(resolver: &dyn Resolver, category: &str) -> PageModel {
    let route = format!("/docs/reference/{category}/");
    let mut children = vec![];
    let mut items = vec![];

    let focus = match category {
        "math" => &LIBRARY.math,
        "calculate" => module(&LIBRARY.global, "calc"),
        _ => &LIBRARY.global,
    };

    let grouped = match category {
        "math" => GROUPS.as_slice(),
        _ => &[],
    };

    // Add functions.
    for (_, value) in focus.scope().iter() {
        let Value::Func(func) = value else { continue };
        let Some(info) = func.info() else { continue };
        if info.category != category {
            continue;
        }

        // Skip grouped functions.
        if grouped
            .iter()
            .flat_map(|group| &group.functions)
            .any(|f| f == info.name)
        {
            continue;
        }

        let subpage = function_page(resolver, &route, func, info);
        items.push(CategoryItem {
            name: info.name.into(),
            route: subpage.route.clone(),
            oneliner: oneliner(info.docs).into(),
            code: true,
        });
        children.push(subpage);
    }

    // Add grouped functions.
    for group in grouped {
        let mut functions = vec![];
        for name in &group.functions {
            let value = focus.get(&name).unwrap();
            let Value::Func(func) = value else { panic!("not a function") };
            let info = func.info().unwrap();
            functions.push(func_model(resolver, func, info));
        }

        let route = format!("{}{}/", route, group.name);
        items.push(CategoryItem {
            name: group.name.clone(),
            route: route.clone(),
            oneliner: oneliner(&group.description).into(),
            code: false,
        });
        children.push(PageModel {
            route,
            title: group.title.clone(),
            description: format!("Documentation for {} group of functions.", group.name),
            part: None,
            body: BodyModel::Funcs(FuncsModel {
                name: group.name.clone(),
                details: Html::markdown(resolver, &group.description),
                functions,
            }),
            children: vec![],
        });
    }

    children.sort_by_cached_key(|child| child.title.clone());
    items.sort_by_cached_key(|item| item.name.clone());

    // Add symbol pages. These are ordered manually.
    if category == "symbols" {
        for module in ["sym", "emoji"] {
            let subpage = symbol_page(resolver, &route, module);
            items.push(CategoryItem {
                name: module.into(),
                route: subpage.route.clone(),
                oneliner: oneliner(details(module)).into(),
                code: true,
            });
            children.push(subpage);
        }
    }

    let name = category.to_title_case();
    PageModel {
        route,
        title: name.clone(),
        description: format!("Documentation for functions related to {name} in Typst."),
        part: None,
        body: BodyModel::Category(CategoryModel {
            name,
            details: Html::markdown(resolver, details(category)),
            kind: match category {
                "symbols" => "Modules",
                _ => "Functions",
            },
            items,
        }),
        children,
    }
}

/// Details about a function.
#[derive(Debug, Serialize)]
pub struct FuncModel {
    pub name: &'static str,
    pub display: &'static str,
    pub oneliner: &'static str,
    pub details: Html,
    pub showable: bool,
    pub params: Vec<ParamModel>,
    pub returns: Vec<&'static str>,
}

/// Details about a group of functions.
#[derive(Debug, Serialize)]
pub struct FuncsModel {
    pub name: String,
    pub details: Html,
    pub functions: Vec<FuncModel>,
}

/// Create a page for a function.
fn function_page(
    resolver: &dyn Resolver,
    parent: &str,
    func: &Func,
    info: &FuncInfo,
) -> PageModel {
    PageModel {
        route: format!("{parent}{}/", urlify(info.name)),
        title: info.display.to_string(),
        description: format!("Documentation for the `{}` function.", info.name),
        part: None,
        body: BodyModel::Func(func_model(resolver, func, info)),
        children: vec![],
    }
}

/// Produce a function's model.
fn func_model(resolver: &dyn Resolver, func: &Func, info: &FuncInfo) -> FuncModel {
    FuncModel {
        name: info.name.into(),
        display: info.display,
        oneliner: oneliner(info.docs),
        details: Html::markdown(resolver, info.docs),
        showable: func.select(None).is_ok() && info.category != "math",
        params: info.params.iter().map(|param| param_model(resolver, param)).collect(),
        returns: info.returns.clone(),
    }
}

/// Details about a function parameter.
#[derive(Debug, Serialize)]
pub struct ParamModel {
    pub name: &'static str,
    pub details: Html,
    pub example: Option<Html>,
    pub types: Vec<&'static str>,
    pub strings: Vec<StrParam>,
    pub positional: bool,
    pub named: bool,
    pub required: bool,
    pub variadic: bool,
    pub settable: bool,
}

/// A specific string that can be passed as an argument.
#[derive(Debug, Serialize)]
pub struct StrParam {
    pub string: String,
    pub details: Html,
}

/// Produce a parameter's model.
fn param_model(resolver: &dyn Resolver, info: &ParamInfo) -> ParamModel {
    let mut types = vec![];
    let mut strings = vec![];
    casts(resolver, &mut types, &mut strings, &info.cast);
    if !strings.is_empty() && !types.contains(&"string") {
        types.push("string");
    }
    types.sort_by_key(|ty| type_index(ty));

    let mut details = info.docs;
    let mut example = None;
    if let Some(mut i) = info.docs.find("```example") {
        while info.docs[..i].ends_with('`') {
            i -= 1;
        }
        details = &info.docs[..i];
        example = Some(&info.docs[i..]);
    }

    ParamModel {
        name: info.name,
        details: Html::markdown(resolver, details),
        example: example.map(|md| Html::markdown(resolver, md)),
        types,
        strings,
        positional: info.positional,
        named: info.named,
        required: info.required,
        variadic: info.variadic,
        settable: info.settable,
    }
}

/// Process cast information into types and strings.
fn casts(
    resolver: &dyn Resolver,
    types: &mut Vec<&'static str>,
    strings: &mut Vec<StrParam>,
    info: &CastInfo,
) {
    match info {
        CastInfo::Any => types.push("any"),
        CastInfo::Value(Value::Str(string), docs) => strings.push(StrParam {
            string: string.to_string(),
            details: Html::markdown(resolver, docs),
        }),
        CastInfo::Value(..) => {}
        CastInfo::Type(ty) => types.push(ty),
        CastInfo::Union(options) => {
            for option in options {
                casts(resolver, types, strings, option);
            }
        }
    }
}

/// A collection of symbols.
#[derive(Debug, Serialize)]
pub struct TypeModel {
    pub name: String,
    pub oneliner: &'static str,
    pub details: Html,
    pub methods: Vec<MethodModel>,
}

/// Details about a built-in method on a type.
#[derive(Debug, Serialize)]
pub struct MethodModel {
    pub name: &'static str,
    pub details: Html,
    pub params: Vec<ParamModel>,
    pub returns: Vec<&'static str>,
}

/// Create a page for the types.
fn types_page(resolver: &dyn Resolver, parent: &str) -> PageModel {
    let route = format!("{parent}types/");
    let mut children = vec![];
    let mut items = vec![];

    for model in type_models(resolver) {
        let route = format!("{route}{}/", urlify(&model.name));
        items.push(CategoryItem {
            name: model.name.clone(),
            route: route.clone(),
            oneliner: model.oneliner.into(),
            code: true,
        });
        children.push(PageModel {
            route,
            title: model.name.to_title_case(),
            description: format!("Documentation for the `{}` type.", model.name),
            part: None,
            body: BodyModel::Type(model),
            children: vec![],
        });
    }

    PageModel {
        route,
        title: "Types".into(),
        description: "Documentation for Typst's built-in types.".into(),
        part: None,
        body: BodyModel::Category(CategoryModel {
            name: "Types".into(),
            details: Html::markdown(resolver, details("types")),
            kind: "Types",
            items,
        }),
        children,
    }
}

/// Produce the types' models.
fn type_models(resolver: &dyn Resolver) -> Vec<TypeModel> {
    let file = SRC.get_file("reference/types.md").unwrap();
    let text = file.contents_utf8().unwrap();

    let mut s = unscanny::Scanner::new(text);
    let mut types = vec![];

    while s.eat_if("# ") {
        let part = s.eat_until("\n# ");
        types.push(type_model(resolver, part));
        s.eat_if('\n');
    }

    types
}

/// Produce a type's model.
fn type_model(resolver: &dyn Resolver, part: &'static str) -> TypeModel {
    let mut s = unscanny::Scanner::new(part);
    let display = s.eat_until('\n').trim();
    let docs = s.eat_until("\n## Methods").trim();

    s.eat_whitespace();

    let mut methods = vec![];
    if s.eat_if("## Methods") {
        s.eat_until("\n### ");
        while s.eat_if("\n### ") {
            methods.push(method_model(resolver, s.eat_until("\n### ")));
        }
    }

    TypeModel {
        name: display.to_lowercase(),
        oneliner: oneliner(docs),
        details: Html::markdown(resolver, docs),
        methods,
    }
}

/// Produce a method's model.
fn method_model(resolver: &dyn Resolver, part: &'static str) -> MethodModel {
    let mut s = unscanny::Scanner::new(part);
    let mut params = vec![];
    let mut returns = vec![];

    let name = s.eat_until('(').trim();
    s.expect("()");
    let docs = s.eat_until("\n- ").trim();

    while s.eat_if("\n- ") {
        let name = s.eat_until(':');
        s.expect(": ");
        let types: Vec<_> =
            s.eat_until(['(', '\n']).split(" or ").map(str::trim).collect();
        if !types.iter().all(|ty| type_index(ty) != usize::MAX) {
            panic!(
                "unknown type in method {} parameter {}",
                name,
                types.iter().find(|ty| type_index(ty) == usize::MAX).unwrap()
            )
        }

        if name == "returns" {
            returns = types;
            continue;
        }

        s.expect('(');

        let mut named = false;
        let mut positional = false;
        let mut required = false;
        let mut variadic = false;
        for part in s.eat_until(')').split(',').map(str::trim) {
            match part {
                "named" => named = true,
                "positional" => positional = true,
                "required" => required = true,
                "variadic" => variadic = true,
                _ => panic!("unknown parameter flag {:?}", part),
            }
        }

        s.expect(')');

        params.push(ParamModel {
            name,
            details: Html::markdown(resolver, s.eat_until("\n- ").trim()),
            example: None,
            types,
            strings: vec![],
            positional,
            named,
            required,
            variadic,
            settable: false,
        });
    }

    MethodModel {
        name,
        details: Html::markdown(resolver, docs),
        params,
        returns,
    }
}

/// A collection of symbols.
#[derive(Debug, Serialize)]
pub struct SymbolsModel {
    pub name: &'static str,
    pub details: Html,
    pub list: Vec<SymbolModel>,
}

/// Details about a symbol.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SymbolModel {
    pub name: String,
    pub shorthand: Option<&'static str>,
    pub codepoint: u32,
    pub accent: bool,
    pub unicode_name: Option<String>,
    pub alternates: Vec<String>,
}

/// Create a page for symbols.
fn symbol_page(resolver: &dyn Resolver, parent: &str, name: &str) -> PageModel {
    let module = &module(&LIBRARY.global, name);

    let mut list = vec![];
    for (name, value) in module.scope().iter() {
        let Value::Symbol(symbol) = value else { continue };
        let complete = |variant: &str| {
            if variant.is_empty() {
                name.into()
            } else {
                format!("{}.{}", name, variant)
            }
        };

        for (variant, c) in symbol.variants() {
            list.push(SymbolModel {
                name: complete(variant),
                shorthand: typst::syntax::ast::Shorthand::LIST
                    .iter()
                    .copied()
                    .find(|&(_, x)| x == c)
                    .map(|(s, _)| s),
                codepoint: c as u32,
                accent: typst::model::combining_accent(c).is_some(),
                unicode_name: unicode_names2::name(c)
                    .map(|s| s.to_string().to_title_case()),
                alternates: symbol
                    .variants()
                    .filter(|(other, _)| other != &variant)
                    .map(|(other, _)| complete(other))
                    .collect(),
            });
        }
    }

    let title = match name {
        "sym" => "General",
        "emoji" => "Emoji",
        _ => unreachable!(),
    };

    PageModel {
        route: format!("{parent}{name}/"),
        title: title.into(),
        description: format!("Documentation for the `{name}` module."),
        part: None,
        body: BodyModel::Symbols(SymbolsModel {
            name: title,
            details: Html::markdown(resolver, details(name)),
            list,
        }),
        children: vec![],
    }
}

/// Data about a collection of functions.
#[derive(Debug, Deserialize)]
struct GroupData {
    name: String,
    title: String,
    functions: Vec<String>,
    description: String,
}

/// Extract a module from another module.
#[track_caller]
fn module<'a>(parent: &'a Module, name: &str) -> &'a Module {
    match parent.scope().get(name) {
        Some(Value::Module(module)) => module,
        _ => panic!("module doesn't contain module `{name}`"),
    }
}

/// Load YAML from a path.
#[track_caller]
fn yaml<T: DeserializeOwned>(path: &str) -> T {
    let file = SRC.get_file(path).unwrap();
    yaml::from_slice(file.contents()).unwrap()
}

/// Load details for an identifying key.
#[track_caller]
fn details(key: &str) -> &str {
    DETAILS
        .get(&yaml::Value::String(key.into()))
        .and_then(|value| value.as_str())
        .unwrap_or_else(|| panic!("missing details for {key}"))
}

/// Turn a title into an URL fragment.
pub fn urlify(title: &str) -> String {
    title
        .chars()
        .map(|c| c.to_ascii_lowercase())
        .map(|c| match c {
            'a'..='z' | '0'..='9' => c,
            _ => '-',
        })
        .collect()
}

/// Extract the first line of documentation.
fn oneliner(docs: &str) -> &str {
    docs.lines().next().unwrap_or_default().into()
}

/// The order of types in the documentation.
fn type_index(ty: &str) -> usize {
    TYPE_ORDER.iter().position(|&v| v == ty).unwrap_or(usize::MAX)
}

const TYPE_ORDER: &[&str] = &[
    "any",
    "none",
    "auto",
    "boolean",
    "integer",
    "float",
    "length",
    "angle",
    "ratio",
    "relative length",
    "fraction",
    "color",
    "string",
    "regex",
    "label",
    "content",
    "array",
    "dictionary",
    "function",
    "arguments",
    "dir",
    "alignment",
    "2d alignment",
    "selector",
    "stroke",
];