typst/src/syntax/token.rs

//! Tokenization.

use crate::length::Length;

/// A minimal semantic entity of source code.
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum Token<'s> {
    /// One or more whitespace characters. The contained `usize` denotes the
    /// number of newlines that were contained in the whitespace.
    Space(usize),

    /// A line comment with inner string contents `//<str>\n`.
    LineComment(&'s str),
    /// A block comment with inner string contents `/*<str>*/`. The comment
    /// can contain nested block comments.
    BlockComment(&'s str),

    /// A left bracket starting a function invocation or body: `[`.
    LeftBracket,
    /// A right bracket ending a function invocation or body: `]`.
    RightBracket,
    /// A left brace indicating the start of content: `{`.
    LeftBrace,
    /// A right brace indicating the end of content: `}`.
    RightBrace,
    /// A left parenthesis in a function header: `(`.
    LeftParen,
    /// A right parenthesis in a function header: `)`.
    RightParen,

    /// A double forward chevron in a function header: `>>`.
    Chain,

    /// A colon in a function header: `:`.
    Colon,
    /// A comma in a function header: `,`.
    Comma,
    /// An equals sign in a function header: `=`.
    Equals,

    /// An identifier in a function header: `center`.
    Ident(&'s str),
    /// A quoted string in a function header: `"..."`.
    Str {
        /// The string inside the quotes.
        ///
        /// _Note_: If the string contains escape sequences these are not yet
        /// applied to be able to just store a string slice here instead of
        /// a String. The escaping is done later in the parser.
        string: &'s str,
        /// Whether the closing quote was present.
        terminated: bool,
    },
    /// A boolean in a function header: `true | false`.
    Bool(bool),
    /// A number in a function header: `3.14`.
    Number(f64),
    /// A length in a function header: `12pt`.
    Length(Length),
    /// A hex value in a function header: `#20d82a`.
    Hex(&'s str),
    /// A plus in a function header, signifying the addition of expressions.
    Plus,
    /// A hyphen in a function header, signifying the subtraction of
    /// expressions.
    Hyphen,
    /// A slash in a function header, signifying the division of expressions.
    Slash,

    /// A star. It can appear in a function header where it signifies the
    /// multiplication of expressions or the body where it modifies the styling.
    Star,
    /// An underscore in body-text.
    Underscore,
    /// A backslash followed by whitespace in text.
    Backslash,

    /// A hashtag token in the body can indicate compute mode or headings.
    Hashtag,

    /// A unicode escape sequence.
    UnicodeEscape {
        /// The escape sequence between two braces.
        sequence: &'s str,
        /// Whether the closing brace was present.
        terminated: bool,
    },

    /// Raw block.
    Raw {
        /// The raw text between the backticks.
        raw: &'s str,
        /// The number of opening backticks.
        backticks: usize,
        /// Whether all closing backticks were present.
        terminated: bool,
    },

    /// Any other consecutive string.
    Text(&'s str),

    /// Things that are not valid in the context they appeared in.
    Invalid(&'s str),
}

impl<'s> Token<'s> {
    /// The natural-language name for this token for use in error messages.
    pub fn name(self) -> &'static str {
        match self {
            Self::Space(_) => "space",
            Self::LineComment(_) => "line comment",
            Self::BlockComment(_) => "block comment",
            Self::LeftBracket => "opening bracket",
            Self::RightBracket => "closing bracket",
            Self::LeftParen => "opening paren",
            Self::RightParen => "closing paren",
            Self::LeftBrace => "opening brace",
            Self::RightBrace => "closing brace",
            Self::Chain => "function chain operator",
            Self::Colon => "colon",
            Self::Comma => "comma",
            Self::Equals => "equals sign",
            Self::Ident(_) => "identifier",
            Self::Str { .. } => "string",
            Self::Bool(_) => "bool",
            Self::Number(_) => "number",
            Self::Length(_) => "length",
            Self::Hex(_) => "hex value",
            Self::Plus => "plus",
            Self::Hyphen => "minus",
            Self::Slash => "slash",
            Self::Star => "star",
            Self::Underscore => "underscore",
            Self::Backslash => "backslash",
            Self::Hashtag => "hashtag",
            Self::UnicodeEscape { .. } => "unicode escape sequence",
            Self::Raw { .. } => "raw block",
            Self::Text(_) => "text",
            Self::Invalid("*/") => "end of block comment",
            Self::Invalid(_) => "invalid token",
        }
    }
}