caisin/typst
1
0
Fork 0
mirror of https://github.com/typst/typst synced 2025-05-13 20:46:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use codex for symbols (#5421)

Browse Source
This commit is contained in:
Malo 2024-11-17 20:08:23 +01:00 committed by GitHub
parent 5672cc2a29
commit 5c37a1cfea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 145 additions and 2647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@ -395,6 +395,11 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "codex"
version = "0.1.0"
source = "git+https://github.com/typst/codex?rev=343a9b1#343a9b199430681ba3ca0e2242097c6419492d55"
[[package]]
name = "color-print"
version = "0.3.6"
@ -2849,6 +2854,7 @@ dependencies = [
"bumpalo",
"chinese-number",
"ciborium",
"codex",
"comemo",
"csv",
"ecow",

1
Cargo.toml
View File

@ -46,6 +46,7 @@ clap = { version = "4.4", features = ["derive", "env", "wrap_help"] }
clap_complete = "4.2.1"
clap_mangen = "0.2.10"
codespan-reporting = "0.11"
codex = { git = "https://github.com/typst/codex", rev = "343a9b1" }
color-print = "0.3.6"
comemo = "0.4"
csv = "1"

4
crates/typst-eval/src/markup.rs
View File

@ -122,7 +122,7 @@ impl Eval for ast::Escape<'_> {
type Output = Value;
fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
Ok(Value::Symbol(Symbol::single(self.get().into())))
Ok(Value::Symbol(Symbol::single(self.get())))
}
}
@ -130,7 +130,7 @@ impl Eval for ast::Shorthand<'_> {
type Output = Value;
fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
Ok(Value::Symbol(Symbol::single(self.get().into())))
Ok(Value::Symbol(Symbol::single(self.get())))
}
}

2
crates/typst-eval/src/math.rs
View File

@ -32,7 +32,7 @@ impl Eval for ast::MathShorthand<'_> {
type Output = Value;
fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
Ok(Value::Symbol(Symbol::single(self.get().into())))
Ok(Value::Symbol(Symbol::single(self.get())))
}
}

1
crates/typst-library/Cargo.toml
View File

@ -23,6 +23,7 @@ bitflags = { workspace = true }
bumpalo = { workspace = true }
chinese-number = { workspace = true }
ciborium = { workspace = true }
codex = { workspace = true }
comemo = { workspace = true }
csv = { workspace = true }
ecow = { workspace = true }

143
crates/typst-library/src/foundations/symbol.rs
View File

@ -1,6 +1,3 @@
#[doc(inline)]
pub use typst_macros::symbols;
use std::cmp::Reverse;
use std::collections::BTreeSet;
use std::fmt::{self, Debug, Display, Formatter, Write};
@ -8,10 +5,10 @@ use std::sync::Arc;
use ecow::{eco_format, EcoString};
use serde::{Serialize, Serializer};
use typst_syntax::{Span, Spanned};
use typst_syntax::{is_ident, Span, Spanned};
use crate::diag::{bail, SourceResult, StrResult};
use crate::foundations::{cast, func, scope, ty, Array, Func};
use crate::foundations::{cast, func, scope, ty, Array, Func, NativeFunc, Repr as _};
/// A Unicode symbol.
///
@ -46,73 +43,90 @@ use crate::foundations::{cast, func, scope, ty, Array, Func};
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct Symbol(Repr);
/// The character of a symbol, possibly with a function.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub struct SymChar(char, Option<fn() -> Func>);
/// The internal representation.
#[derive(Clone, Eq, PartialEq, Hash)]
enum Repr {
Single(SymChar),
Const(&'static [(&'static str, SymChar)]),
Multi(Arc<(List, EcoString)>),
/// A native symbol that has no named variant.
Single(char),
/// A native symbol with multiple named variants.
Complex(&'static [(&'static str, char)]),
/// A symbol with multiple named variants, where some modifiers may have
/// been applied. Also used for symbols defined at runtime by the user with
/// no modifier applied.
Modified(Arc<(List, EcoString)>),
}
/// A collection of symbols.
#[derive(Clone, Eq, PartialEq, Hash)]
enum List {
Static(&'static [(&'static str, SymChar)]),
Runtime(Box<[(EcoString, SymChar)]>),
Static(&'static [(&'static str, char)]),
Runtime(Box<[(EcoString, char)]>),
}
impl Symbol {
/// Create a new symbol from a single character.
pub const fn single(c: SymChar) -> Self {
pub const fn single(c: char) -> Self {
Self(Repr::Single(c))
}
/// Create a symbol with a static variant list.
#[track_caller]
pub const fn list(list: &'static [(&'static str, SymChar)]) -> Self {
pub const fn list(list: &'static [(&'static str, char)]) -> Self {
debug_assert!(!list.is_empty());
Self(Repr::Const(list))
Self(Repr::Complex(list))
}
/// Create a symbol with a runtime variant list.
#[track_caller]
pub fn runtime(list: Box<[(EcoString, SymChar)]>) -> Self {
pub fn runtime(list: Box<[(EcoString, char)]>) -> Self {
debug_assert!(!list.is_empty());
Self(Repr::Multi(Arc::new((List::Runtime(list), EcoString::new()))))
Self(Repr::Modified(Arc::new((List::Runtime(list), EcoString::new()))))
}
/// Get the symbol's char.
/// Get the symbol's character.
pub fn get(&self) -> char {
self.sym().char()
}
/// Resolve the symbol's `SymChar`.
pub fn sym(&self) -> SymChar {
match &self.0 {
Repr::Single(c) => *c,
Repr::Const(_) => find(self.variants(), "").unwrap(),
Repr::Multi(arc) => find(self.variants(), &arc.1).unwrap(),
Repr::Complex(_) => find(self.variants(), "").unwrap(),
Repr::Modified(arc) => find(self.variants(), &arc.1).unwrap(),
}
}
/// Try to get the function associated with the symbol, if any.
pub fn func(&self) -> StrResult<Func> {
self.sym()
.func()
.ok_or_else(|| eco_format!("symbol {self} is not callable"))
match self.get() {
'⌈' => Ok(crate::math::ceil::func()),
'⌊' => Ok(crate::math::floor::func()),
'–' => Ok(crate::math::accent::dash::func()),
'⋅' | '\u{0307}' => Ok(crate::math::accent::dot::func()),
'¨' => Ok(crate::math::accent::dot_double::func()),
'\u{20db}' => Ok(crate::math::accent::dot_triple::func()),
'\u{20dc}' => Ok(crate::math::accent::dot_quad::func()),
'∼' => Ok(crate::math::accent::tilde::func()),
'´' => Ok(crate::math::accent::acute::func()),
'˝' => Ok(crate::math::accent::acute_double::func()),
'˘' => Ok(crate::math::accent::breve::func()),
'ˇ' => Ok(crate::math::accent::caron::func()),
'^' => Ok(crate::math::accent::hat::func()),
'`' => Ok(crate::math::accent::grave::func()),
'¯' => Ok(crate::math::accent::macron::func()),
'○' => Ok(crate::math::accent::circle::func()),
'→' => Ok(crate::math::accent::arrow::func()),
'←' => Ok(crate::math::accent::arrow_l::func()),
'↔' => Ok(crate::math::accent::arrow_l_r::func()),
'⇀' => Ok(crate::math::accent::harpoon::func()),
'↼' => Ok(crate::math::accent::harpoon_lt::func()),
_ => bail!("symbol {self} is not callable"),
}
}
/// Apply a modifier to the symbol.
pub fn modified(mut self, modifier: &str) -> StrResult<Self> {
if let Repr::Const(list) = self.0 {
self.0 = Repr::Multi(Arc::new((List::Static(list), EcoString::new())));
if let Repr::Complex(list) = self.0 {
self.0 = Repr::Modified(Arc::new((List::Static(list), EcoString::new())));
}
if let Repr::Multi(arc) = &mut self.0 {
if let Repr::Modified(arc) = &mut self.0 {
let (list, modifiers) = Arc::make_mut(arc);
if !modifiers.is_empty() {
modifiers.push('.');
@ -127,11 +141,11 @@ impl Symbol {
}
/// The characters that are covered by this symbol.
pub fn variants(&self) -> impl Iterator<Item = (&str, SymChar)> {
pub fn variants(&self) -> impl Iterator<Item = (&str, char)> {
match &self.0 {
Repr::Single(c) => Variants::Single(Some(*c).into_iter()),
Repr::Const(list) => Variants::Static(list.iter()),
Repr::Multi(arc) => arc.0.variants(),
Repr::Complex(list) => Variants::Static(list.iter()),
Repr::Modified(arc) => arc.0.variants(),
}
}
@ -139,7 +153,7 @@ impl Symbol {
pub fn modifiers(&self) -> impl Iterator<Item = &str> + '_ {
let mut set = BTreeSet::new();
let modifiers = match &self.0 {
Repr::Multi(arc) => arc.1.as_str(),
Repr::Modified(arc) => arc.1.as_str(),
_ => "",
};
for modifier in self.variants().flat_map(|(name, _)| name.split('.')) {
@ -192,7 +206,14 @@ impl Symbol {
if list.iter().any(|(prev, _)| &v.0 == prev) {
bail!(span, "duplicate variant");
}
list.push((v.0, SymChar::pure(v.1)));
if !v.0.is_empty() {
for modifier in v.0.split('.') {
if !is_ident(modifier) {
bail!(span, "invalid symbol modifier: {}", modifier.repr());
}
}
}
list.push((v.0, v.1));
}
Ok(Symbol::runtime(list.into_boxed_slice()))
}
@ -204,40 +225,12 @@ impl Display for Symbol {
}
}
impl SymChar {
/// Create a symbol character without a function.
pub const fn pure(c: char) -> Self {
Self(c, None)
}
/// Create a symbol character with a function.
pub const fn with_func(c: char, func: fn() -> Func) -> Self {
Self(c, Some(func))
}
/// Get the character of the symbol.
pub const fn char(&self) -> char {
self.0
}
/// Get the function associated with the symbol.
pub fn func(&self) -> Option<Func> {
self.1.map(|f| f())
}
}
impl From<char> for SymChar {
fn from(c: char) -> Self {
SymChar(c, None)
}
}
impl Debug for Repr {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
match self {
Self::Single(c) => Debug::fmt(c, f),
Self::Const(list) => list.fmt(f),
Self::Multi(lists) => lists.fmt(f),
Self::Complex(list) => list.fmt(f),
Self::Modified(lists) => lists.fmt(f),
}
}
}
@ -286,20 +279,20 @@ cast! {
let mut iter = array.into_iter();
match (iter.next(), iter.next(), iter.next()) {
(Some(a), Some(b), None) => Self(a.cast()?, b.cast()?),
_ => Err("point array must contain exactly two entries")?,
_ => Err("variant array must contain exactly two entries")?,
}
},
}
/// Iterator over variants.
enum Variants<'a> {
Single(std::option::IntoIter<SymChar>),
Static(std::slice::Iter<'static, (&'static str, SymChar)>),
Runtime(std::slice::Iter<'a, (EcoString, SymChar)>),
Single(std::option::IntoIter<char>),
Static(std::slice::Iter<'static, (&'static str, char)>),
Runtime(std::slice::Iter<'a, (EcoString, char)>),
}
impl<'a> Iterator for Variants<'a> {
type Item = (&'a str, SymChar);
type Item = (&'a str, char);
fn next(&mut self) -> Option<Self::Item> {
match self {
@ -312,9 +305,9 @@ impl<'a> Iterator for Variants<'a> {
/// Find the best symbol from the list.
fn find<'a>(
variants: impl Iterator<Item = (&'a str, SymChar)>,
variants: impl Iterator<Item = (&'a str, char)>,
modifiers: &str,
) -> Option<SymChar> {
) -> Option<char> {
let mut best = None;
let mut best_score = None;

4
crates/typst-library/src/math/mod.rs
View File

@ -207,9 +207,7 @@ pub fn module() -> Module {
math.define("wide", HElem::new(WIDE.into()).pack());
// Symbols.
for (name, symbol) in crate::symbols::SYM {
math.define(*name, symbol.clone());
}
crate::symbols::define_math(&mut math);
Module::new("math", math)
}

49
crates/typst-library/src/symbols.rs Normal file
View File

@ -0,0 +1,49 @@
//! Modifiable symbols.
use crate::foundations::{category, Category, Module, Scope, Symbol, Value};
/// These two modules give names to symbols and emoji to make them easy to
/// insert with a normal keyboard. Alternatively, you can also always directly
/// enter Unicode symbols into your text and formulas. In addition to the
/// symbols listed below, math mode defines `dif` and `Dif`. These are not
/// normal symbol values because they also affect spacing and font style.
#[category]
pub static SYMBOLS: Category;
impl From<codex::Module> for Scope {
fn from(module: codex::Module) -> Scope {
let mut scope = Self::new();
extend_scope_from_codex_module(&mut scope, module);
scope
}
}
impl From<codex::Symbol> for Symbol {
fn from(symbol: codex::Symbol) -> Self {
match symbol {
codex::Symbol::Single(c) => Symbol::single(c),
codex::Symbol::Multi(list) => Symbol::list(list),
}
}
}
fn extend_scope_from_codex_module(scope: &mut Scope, module: codex::Module) {
for (name, definition) in module.iter() {
let value = match definition {
codex::Def::Symbol(s) => Value::Symbol(s.into()),
codex::Def::Module(m) => Value::Module(Module::new(name, m.into())),
};
scope.define(name, value);
}
}
/// Hook up all `symbol` definitions.
pub(super) fn define(global: &mut Scope) {
global.category(SYMBOLS);
extend_scope_from_codex_module(global, codex::ROOT);
}
/// Hook up all math `symbol` definitions, i.e., elements of the `sym` module.
pub(super) fn define_math(math: &mut Scope) {
extend_scope_from_codex_module(math, codex::SYM);
}

1363
crates/typst-library/src/symbols/emoji.rs
View File

File diff suppressed because it is too large Load Diff

24
crates/typst-library/src/symbols/mod.rs
View File

@ -1,24 +0,0 @@
//! Modifiable symbols.
mod emoji;
mod sym;
pub use self::emoji::*;
pub use self::sym::*;
use crate::foundations::{category, Category, Scope};
/// These two modules give names to symbols and emoji to make them easy to
/// insert with a normal keyboard. Alternatively, you can also always directly
/// enter Unicode symbols into your text and formulas. In addition to the
/// symbols listed below, math mode defines `dif` and `Dif`. These are not
/// normal symbol values because they also affect spacing and font style.
#[category]
pub static SYMBOLS: Category;
/// Hook up all `symbol` definitions.
pub(super) fn define(global: &mut Scope) {
global.category(SYMBOLS);
global.define_module(sym());
global.define_module(emoji());
}

1000
crates/typst-library/src/symbols/sym.rs
View File

File diff suppressed because it is too large Load Diff

47
crates/typst-macros/src/lib.rs
View File

@ -9,7 +9,6 @@ mod category;
mod elem;
mod func;
mod scope;
mod symbols;
mod time;
mod ty;
@ -338,52 +337,6 @@ pub fn derive_cast(item: BoundaryStream) -> BoundaryStream {
.into()
}
/// Defines a list of `Symbol`s.
///
/// The `#[call(path)]` attribute can be used to specify a function to call when
/// the symbol is invoked. The function must be `NativeFunc`.
///
/// ```ignore
/// const EMOJI: &[(&str, Symbol)] = symbols! {
/// // A plain symbol without modifiers.
/// abacus: '🧮',
///
/// // A symbol with a modifierless default and one modifier.
/// alien: ['👽', monster: '👾'],
///
/// // A symbol where each variant has a modifier. The first one will be
/// // the default.
/// clock: [one: '🕐', two: '🕑', ...],
///
/// // A callable symbol without modifiers.
/// breve: #[call(crate::math::breve)] '˘',
///
/// // A callable symbol with a modifierless default and one modifier.
/// acute: [
/// #[call(crate::math::acute)] '´',
/// double: '˝',
/// ],
///
/// // A callable symbol where each variant has a modifier.
/// arrow: [
/// #[call(crate::math::arrow)] r: '→',
/// r.long.bar: '⟼',
/// #[call(crate::math::arrow_l)] l: '←',
/// l.long.bar: '⟻',
/// ],
/// }
/// ```
///
/// _Note:_ While this could use `macro_rules!` instead of a proc-macro, it was
/// horribly slow in rust-analyzer. The underlying cause might be
/// [this issue](https://github.com/rust-lang/rust-analyzer/issues/11108).
#[proc_macro]
pub fn symbols(stream: BoundaryStream) -> BoundaryStream {
symbols::symbols(stream.into())
.unwrap_or_else(|err| err.to_compile_error())
.into()
}
/// Times function invocations.
///
/// When tracing is enabled in the typst-cli, this macro will record the

129
crates/typst-macros/src/symbols.rs
View File

@ -1,129 +0,0 @@
use proc_macro2::TokenStream;
use quote::quote;
use syn::ext::IdentExt;
use syn::parse::{Parse, ParseStream, Parser};
use syn::punctuated::Punctuated;
use syn::{Ident, LitChar, Path, Result, Token};
use crate::util::foundations;
/// Expand the `symbols!` macro.
pub fn symbols(stream: TokenStream) -> Result<TokenStream> {
let list: Punctuated<Symbol, Token![,]> =
Punctuated::parse_terminated.parse2(stream)?;
let pairs = list.iter().map(|symbol| {
let name = symbol.name.to_string();
let kind = match &symbol.kind {
Kind::Single(c, h) => {
let symbol = construct_sym_char(c, h);
quote! { #foundations::Symbol::single(#symbol), }
}
Kind::Multiple(variants) => {
let variants = variants.iter().map(|variant| {
let name = &variant.name;
let c = &variant.c;
let symbol = construct_sym_char(c, &variant.handler);
quote! { (#name, #symbol) }
});
quote! {
#foundations::Symbol::list(&[#(#variants),*])
}
}
};
quote! { (#name, #kind) }
});
Ok(quote! { &[#(#pairs),*] })
}
fn construct_sym_char(ch: &LitChar, handler: &Handler) -> TokenStream {
match &handler.0 {
None => quote! { #foundations::SymChar::pure(#ch), },
Some(path) => quote! {
#foundations::SymChar::with_func(
#ch,
<#path as ::typst_library::foundations::NativeFunc>::func,
),
},
}
}
struct Symbol {
name: syn::Ident,
kind: Kind,
}
enum Kind {
Single(syn::LitChar, Handler),
Multiple(Punctuated<Variant, Token![,]>),
}
struct Variant {
name: String,
c: syn::LitChar,
handler: Handler,
}
struct Handler(Option<Path>);
impl Parse for Symbol {
fn parse(input: ParseStream) -> Result<Self> {
let name = input.call(Ident::parse_any)?;
input.parse::<Token![:]>()?;
let kind = input.parse()?;
Ok(Self { name, kind })
}
}
impl Parse for Kind {
fn parse(input: ParseStream) -> Result<Self> {
let handler = input.parse::<Handler>()?;
if input.peek(syn::LitChar) {
Ok(Self::Single(input.parse()?, handler))
} else {
if handler.0.is_some() {
return Err(input.error("unexpected handler"));
}
let content;
syn::bracketed!(content in input);
Ok(Self::Multiple(Punctuated::parse_terminated(&content)?))
}
}
}
impl Parse for Variant {
fn parse(input: ParseStream) -> Result<Self> {
let mut name = String::new();
let handler = input.parse::<Handler>()?;
if input.peek(syn::Ident::peek_any) {
name.push_str(&input.call(Ident::parse_any)?.to_string());
while input.peek(Token![.]) {
input.parse::<Token![.]>()?;
name.push('.');
name.push_str(&input.call(Ident::parse_any)?.to_string());
}
input.parse::<Token![:]>()?;
}
let c = input.parse()?;
Ok(Self { name, c, handler })
}
}
impl Parse for Handler {
fn parse(input: ParseStream) -> Result<Self> {
let Ok(attrs) = input.call(syn::Attribute::parse_outer) else {
return Ok(Self(None));
};
let handler = attrs
.iter()
.find_map(|attr| {
if attr.path().is_ident("call") {
if let Ok(path) = attr.parse_args::<Path>() {
return Some(Self(Some(path)));
}
}
None
})
.unwrap_or(Self(None));
Ok(handler)
}
}

6
docs/src/lib.rs
View File

@ -671,15 +671,15 @@ fn symbols_model(resolver: &dyn Resolver, group: &GroupData) -> SymbolsModel {
for (variant, c) in symbol.variants() {
let shorthand = |list: &[(&'static str, char)]| {
list.iter().copied().find(|&(_, x)| x == c.char()).map(|(s, _)| s)
list.iter().copied().find(|&(_, x)| x == c).map(|(s, _)| s)
};
list.push(SymbolModel {
name: complete(variant),
markup_shorthand: shorthand(typst::syntax::ast::Shorthand::LIST),
math_shorthand: shorthand(typst::syntax::ast::MathShorthand::LIST),
codepoint: c.char() as _,
accent: typst::math::Accent::combine(c.char()).is_some(),
codepoint: c as _,
accent: typst::math::Accent::combine(c).is_some(),
alternates: symbol
.variants()
.filter(|(other, _)| other != &variant)

13
tests/suite/symbols/symbol.typ
View File

@ -33,6 +33,19 @@
// Error: 2-10 expected at least one variant
#symbol()
--- symbol-constructor-invalid-modifier ---
// Error: 2:3-2:24 invalid symbol modifier: " id!"
#symbol(
("invalid. id!", "x")
)
--- symbol-constructor-duplicate-variant ---
// Error: 3:3-3:29 duplicate variant
#symbol(
("duplicate.variant", "x"),
("duplicate.variant", "y"),
)
--- symbol-unknown-modifier ---
// Error: 13-20 unknown symbol modifier
#emoji.face.garbage