mirror of
https://github.com/typst/typst
synced 2025-06-28 16:22:53 +08:00
Add typst-kit crate (#4540)
This commit is contained in:
parent
810491c9d3
commit
672f6e5f97
25
Cargo.lock
generated
25
Cargo.lock
generated
@ -2670,15 +2670,11 @@ dependencies = [
|
||||
"comemo",
|
||||
"dirs",
|
||||
"ecow",
|
||||
"env_proxy",
|
||||
"flate2",
|
||||
"fontdb",
|
||||
"fs_extra",
|
||||
"native-tls",
|
||||
"notify",
|
||||
"once_cell",
|
||||
"open",
|
||||
"openssl",
|
||||
"parking_lot",
|
||||
"pathdiff",
|
||||
"rayon",
|
||||
@ -2694,6 +2690,7 @@ dependencies = [
|
||||
"toml",
|
||||
"typst",
|
||||
"typst-assets",
|
||||
"typst-kit",
|
||||
"typst-macros",
|
||||
"typst-pdf",
|
||||
"typst-render",
|
||||
@ -2760,6 +2757,26 @@ dependencies = [
|
||||
"unscanny",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typst-kit"
|
||||
version = "0.11.0"
|
||||
dependencies = [
|
||||
"dirs",
|
||||
"ecow",
|
||||
"env_proxy",
|
||||
"flate2",
|
||||
"fontdb",
|
||||
"native-tls",
|
||||
"once_cell",
|
||||
"openssl",
|
||||
"tar",
|
||||
"typst",
|
||||
"typst-assets",
|
||||
"typst-timing",
|
||||
"typst-utils",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typst-macros"
|
||||
version = "0.11.0"
|
||||
|
@ -19,6 +19,7 @@ readme = "README.md"
|
||||
typst = { path = "crates/typst", version = "0.11.0" }
|
||||
typst-cli = { path = "crates/typst-cli", version = "0.11.0" }
|
||||
typst-ide = { path = "crates/typst-ide", version = "0.11.0" }
|
||||
typst-kit = { path = "crates/typst-kit", version = "0.11.0" }
|
||||
typst-macros = { path = "crates/typst-macros", version = "0.11.0" }
|
||||
typst-pdf = { path = "crates/typst-pdf", version = "0.11.0" }
|
||||
typst-render = { path = "crates/typst-render", version = "0.11.0" }
|
||||
|
@ -20,6 +20,7 @@ doc = false
|
||||
[dependencies]
|
||||
typst = { workspace = true }
|
||||
typst-assets = { workspace = true, features = ["fonts"] }
|
||||
typst-kit = { workspace = true }
|
||||
typst-macros = { workspace = true }
|
||||
typst-pdf = { workspace = true }
|
||||
typst-render = { workspace = true }
|
||||
@ -31,9 +32,6 @@ codespan-reporting = { workspace = true }
|
||||
comemo = { workspace = true }
|
||||
dirs = { workspace = true }
|
||||
ecow = { workspace = true }
|
||||
env_proxy = { workspace = true }
|
||||
flate2 = { workspace = true }
|
||||
fontdb = { workspace = true, features = ["memmap", "fontconfig"] }
|
||||
fs_extra = { workspace = true }
|
||||
native-tls = { workspace = true }
|
||||
notify = { workspace = true }
|
||||
@ -56,11 +54,6 @@ ureq = { workspace = true }
|
||||
xz2 = { workspace = true, optional = true }
|
||||
zip = { workspace = true, optional = true }
|
||||
|
||||
# Explicitly depend on OpenSSL if applicable, so that we can add the
|
||||
# `openssl/vendored` feature to it if `vendor-openssl` is enabled.
|
||||
[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies]
|
||||
openssl = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
chrono = { workspace = true }
|
||||
clap = { workspace = true, features = ["string"] }
|
||||
@ -71,17 +64,14 @@ semver = { workspace = true }
|
||||
[features]
|
||||
default = ["embed-fonts"]
|
||||
|
||||
# Embeds some fonts into the binary:
|
||||
# - For text: Linux Libertine, New Computer Modern
|
||||
# - For math: New Computer Modern Math
|
||||
# - For code: Deja Vu Sans Mono
|
||||
embed-fonts = []
|
||||
# Embeds some fonts into the binary, see typst-kit
|
||||
embed-fonts = ["typst-kit/embed-fonts"]
|
||||
|
||||
# Permits the CLI to update itself without a package manager.
|
||||
self-update = ["dep:self-replace", "dep:xz2", "dep:zip"]
|
||||
|
||||
# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds.
|
||||
vendor-openssl = ["openssl/vendored"]
|
||||
vendor-openssl = ["typst-kit/vendor-openssl"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
@ -1,205 +1,90 @@
|
||||
// Acknowledgement:
|
||||
// Closely modelled after rustup's `DownloadTracker`.
|
||||
// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::io::{self, ErrorKind, Read, Write};
|
||||
use std::sync::Arc;
|
||||
use std::fmt::Display;
|
||||
use std::io;
|
||||
use std::io::Write;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use native_tls::{Certificate, TlsConnector};
|
||||
use once_cell::sync::OnceCell;
|
||||
use ureq::Response;
|
||||
use codespan_reporting::term;
|
||||
use codespan_reporting::term::termcolor::WriteColor;
|
||||
use typst_kit::download::{DownloadState, Downloader, Progress};
|
||||
|
||||
use crate::terminal;
|
||||
use crate::terminal::{self, TermOut};
|
||||
use crate::ARGS;
|
||||
|
||||
/// Keep track of this many download speed samples.
|
||||
const SPEED_SAMPLES: usize = 5;
|
||||
/// Prints download progress by writing `downloading {0}` followed by repeatedly
|
||||
/// updating the last terminal line.
|
||||
pub struct PrintDownload<T>(pub T);
|
||||
|
||||
/// Load a certificate from the file system if the `--cert` argument or
|
||||
/// `TYPST_CERT` environment variable is present. The certificate is cached for
|
||||
/// efficiency.
|
||||
///
|
||||
/// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
|
||||
/// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
|
||||
/// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
|
||||
fn cert() -> Option<Result<&'static Certificate, io::Error>> {
|
||||
static CERT: OnceCell<Certificate> = OnceCell::new();
|
||||
crate::ARGS.cert.as_ref().map(|path| {
|
||||
CERT.get_or_try_init(|| {
|
||||
let pem = std::fs::read(path)?;
|
||||
Certificate::from_pem(&pem).map_err(io::Error::other)
|
||||
})
|
||||
})
|
||||
}
|
||||
impl<T: Display> Progress for PrintDownload<T> {
|
||||
fn print_start(&mut self) {
|
||||
// Print that a package downloading is happening.
|
||||
let styles = term::Styles::default();
|
||||
|
||||
/// Download binary data and display its progress.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn download_with_progress(url: &str) -> Result<Vec<u8>, ureq::Error> {
|
||||
let response = download(url)?;
|
||||
Ok(RemoteReader::from_response(response).download()?)
|
||||
}
|
||||
let mut out = terminal::out();
|
||||
let _ = out.set_color(&styles.header_help);
|
||||
let _ = write!(out, "downloading");
|
||||
|
||||
/// Download from a URL.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn download(url: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
let mut builder = ureq::AgentBuilder::new();
|
||||
let mut tls = TlsConnector::builder();
|
||||
|
||||
// Set user agent.
|
||||
builder = builder.user_agent(concat!("typst/", env!("CARGO_PKG_VERSION")));
|
||||
|
||||
// Get the network proxy config from the environment and apply it.
|
||||
if let Some(proxy) = env_proxy::for_url_str(url)
|
||||
.to_url()
|
||||
.and_then(|url| ureq::Proxy::new(url).ok())
|
||||
{
|
||||
builder = builder.proxy(proxy);
|
||||
let _ = out.reset();
|
||||
let _ = writeln!(out, " {}", self.0);
|
||||
}
|
||||
|
||||
// Apply a custom CA certificate if present.
|
||||
if let Some(cert) = cert() {
|
||||
tls.add_root_certificate(cert?.clone());
|
||||
fn print_progress(&mut self, state: &DownloadState) {
|
||||
let mut out = terminal::out();
|
||||
let _ = out.clear_last_line();
|
||||
let _ = display_download_progress(&mut out, state);
|
||||
}
|
||||
|
||||
// Configure native TLS.
|
||||
let connector =
|
||||
tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
||||
builder = builder.tls_connector(Arc::new(connector));
|
||||
|
||||
builder.build().get(url).call()
|
||||
fn print_finish(&mut self, state: &DownloadState) {
|
||||
let mut out = terminal::out();
|
||||
let _ = display_download_progress(&mut out, state);
|
||||
let _ = writeln!(out);
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper around [`ureq::Response`] that reads the response body in chunks
|
||||
/// over a websocket and displays statistics about its progress.
|
||||
///
|
||||
/// Downloads will _never_ fail due to statistics failing to print, print errors
|
||||
/// are silently ignored.
|
||||
struct RemoteReader {
|
||||
reader: Box<dyn Read + Send + Sync + 'static>,
|
||||
content_len: Option<usize>,
|
||||
total_downloaded: usize,
|
||||
downloaded_this_sec: usize,
|
||||
downloaded_last_few_secs: VecDeque<usize>,
|
||||
start_time: Instant,
|
||||
last_print: Option<Instant>,
|
||||
/// Returns a new downloader.
|
||||
pub fn downloader() -> Downloader {
|
||||
let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION"));
|
||||
match ARGS.cert.clone() {
|
||||
Some(cert) => Downloader::with_path(user_agent, cert),
|
||||
None => Downloader::new(user_agent),
|
||||
}
|
||||
}
|
||||
|
||||
impl RemoteReader {
|
||||
/// Wraps a [`ureq::Response`] and prepares it for downloading.
|
||||
///
|
||||
/// The 'Content-Length' header is used as a size hint for read
|
||||
/// optimization, if present.
|
||||
pub fn from_response(response: Response) -> Self {
|
||||
let content_len: Option<usize> = response
|
||||
.header("Content-Length")
|
||||
.and_then(|header| header.parse().ok());
|
||||
/// Compile and format several download statistics and make and attempt at
|
||||
/// displaying them on standard error.
|
||||
pub fn display_download_progress(
|
||||
out: &mut TermOut,
|
||||
state: &DownloadState,
|
||||
) -> io::Result<()> {
|
||||
let sum: usize = state.bytes_per_second.iter().sum();
|
||||
let len = state.bytes_per_second.len();
|
||||
let speed = if len > 0 { sum / len } else { state.content_len.unwrap_or(0) };
|
||||
|
||||
Self {
|
||||
reader: response.into_reader(),
|
||||
content_len,
|
||||
total_downloaded: 0,
|
||||
downloaded_this_sec: 0,
|
||||
downloaded_last_few_secs: VecDeque::with_capacity(SPEED_SAMPLES),
|
||||
start_time: Instant::now(),
|
||||
last_print: None,
|
||||
let total_downloaded = as_bytes_unit(state.total_downloaded);
|
||||
let speed_h = as_throughput_unit(speed);
|
||||
let elapsed = time_suffix(Instant::now().saturating_duration_since(state.start_time));
|
||||
|
||||
match state.content_len {
|
||||
Some(content_len) => {
|
||||
let percent = (state.total_downloaded as f64 / content_len as f64) * 100.;
|
||||
let remaining = content_len - state.total_downloaded;
|
||||
|
||||
let download_size = as_bytes_unit(content_len);
|
||||
let eta = time_suffix(Duration::from_secs(if speed == 0 {
|
||||
0
|
||||
} else {
|
||||
(remaining / speed) as u64
|
||||
}));
|
||||
writeln!(
|
||||
out,
|
||||
"{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
/// Download the bodies content as raw bytes while attempting to print
|
||||
/// download statistics to standard error. Download progress gets displayed
|
||||
/// and updated every second.
|
||||
///
|
||||
/// These statistics will never prevent a download from completing, errors
|
||||
/// are silently ignored.
|
||||
pub fn download(mut self) -> io::Result<Vec<u8>> {
|
||||
let mut buffer = vec![0; 8192];
|
||||
let mut data = match self.content_len {
|
||||
Some(content_len) => Vec::with_capacity(content_len),
|
||||
None => Vec::with_capacity(8192),
|
||||
};
|
||||
|
||||
loop {
|
||||
let read = match self.reader.read(&mut buffer) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => n,
|
||||
// If the data is not yet ready but will be available eventually
|
||||
// keep trying until we either get an actual error, receive data
|
||||
// or an Ok(0).
|
||||
Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
data.extend(&buffer[..read]);
|
||||
|
||||
let last_printed = match self.last_print {
|
||||
Some(prev) => prev,
|
||||
None => {
|
||||
let current_time = Instant::now();
|
||||
self.last_print = Some(current_time);
|
||||
current_time
|
||||
}
|
||||
};
|
||||
let elapsed = Instant::now().saturating_duration_since(last_printed);
|
||||
|
||||
self.total_downloaded += read;
|
||||
self.downloaded_this_sec += read;
|
||||
|
||||
if elapsed >= Duration::from_secs(1) {
|
||||
if self.downloaded_last_few_secs.len() == SPEED_SAMPLES {
|
||||
self.downloaded_last_few_secs.pop_back();
|
||||
}
|
||||
|
||||
self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
|
||||
self.downloaded_this_sec = 0;
|
||||
|
||||
terminal::out().clear_last_line()?;
|
||||
self.display()?;
|
||||
self.last_print = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
self.display()?;
|
||||
writeln!(&mut terminal::out())?;
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// Compile and format several download statistics and make an attempt at
|
||||
/// displaying them on standard error.
|
||||
fn display(&mut self) -> io::Result<()> {
|
||||
let sum: usize = self.downloaded_last_few_secs.iter().sum();
|
||||
let len = self.downloaded_last_few_secs.len();
|
||||
let speed = if len > 0 { sum / len } else { self.content_len.unwrap_or(0) };
|
||||
|
||||
let total_downloaded = as_bytes_unit(self.total_downloaded);
|
||||
let speed_h = as_throughput_unit(speed);
|
||||
let elapsed =
|
||||
time_suffix(Instant::now().saturating_duration_since(self.start_time));
|
||||
|
||||
match self.content_len {
|
||||
Some(content_len) => {
|
||||
let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
|
||||
let remaining = content_len - self.total_downloaded;
|
||||
|
||||
let download_size = as_bytes_unit(content_len);
|
||||
let eta = time_suffix(Duration::from_secs(if speed == 0 {
|
||||
0
|
||||
} else {
|
||||
(remaining / speed) as u64
|
||||
}));
|
||||
writeln!(
|
||||
terminal::out(),
|
||||
"{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
|
||||
)?;
|
||||
}
|
||||
None => writeln!(
|
||||
terminal::out(),
|
||||
"Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
|
||||
)?,
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
None => writeln!(
|
||||
out,
|
||||
"Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
|
||||
)?,
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Append a unit-of-time suffix.
|
||||
|
@ -1,20 +1,16 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use fontdb::{Database, Source};
|
||||
use typst::diag::StrResult;
|
||||
use typst::text::{Font, FontBook, FontInfo, FontVariant};
|
||||
use typst_timing::TimingScope;
|
||||
use typst::text::FontVariant;
|
||||
use typst_kit::fonts::Fonts;
|
||||
|
||||
use crate::args::FontsCommand;
|
||||
|
||||
/// Execute a font listing command.
|
||||
pub fn fonts(command: &FontsCommand) -> StrResult<()> {
|
||||
let mut searcher = FontSearcher::new();
|
||||
searcher.search(&command.font_args.font_paths, command.font_args.ignore_system_fonts);
|
||||
let fonts = Fonts::searcher()
|
||||
.include_system_fonts(!command.font_args.ignore_system_fonts)
|
||||
.search_with(&command.font_args.font_paths);
|
||||
|
||||
for (name, infos) in searcher.book.families() {
|
||||
for (name, infos) in fonts.book.families() {
|
||||
println!("{name}");
|
||||
if command.variants {
|
||||
for info in infos {
|
||||
@ -26,99 +22,3 @@ pub fn fonts(command: &FontsCommand) -> StrResult<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Searches for fonts.
|
||||
pub struct FontSearcher {
|
||||
/// Metadata about all discovered fonts.
|
||||
pub book: FontBook,
|
||||
/// Slots that the fonts are loaded into.
|
||||
pub fonts: Vec<FontSlot>,
|
||||
}
|
||||
|
||||
/// Holds details about the location of a font and lazily the font itself.
|
||||
pub struct FontSlot {
|
||||
/// The path at which the font can be found on the system.
|
||||
path: PathBuf,
|
||||
/// The index of the font in its collection. Zero if the path does not point
|
||||
/// to a collection.
|
||||
index: u32,
|
||||
/// The lazily loaded font.
|
||||
font: OnceLock<Option<Font>>,
|
||||
}
|
||||
|
||||
impl FontSlot {
|
||||
/// Get the font for this slot.
|
||||
pub fn get(&self) -> Option<Font> {
|
||||
self.font
|
||||
.get_or_init(|| {
|
||||
let _scope = TimingScope::new("load font", None);
|
||||
let data = fs::read(&self.path).ok()?.into();
|
||||
Font::new(data, self.index)
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl FontSearcher {
|
||||
/// Create a new, empty system searcher.
|
||||
pub fn new() -> Self {
|
||||
Self { book: FontBook::new(), fonts: vec![] }
|
||||
}
|
||||
|
||||
/// Search everything that is available.
|
||||
pub fn search(&mut self, font_paths: &[PathBuf], ignore_system_fonts: bool) {
|
||||
let mut db = Database::new();
|
||||
|
||||
// Font paths have highest priority.
|
||||
for path in font_paths {
|
||||
db.load_fonts_dir(path);
|
||||
}
|
||||
|
||||
if !ignore_system_fonts {
|
||||
// System fonts have second priority.
|
||||
db.load_system_fonts();
|
||||
}
|
||||
|
||||
for face in db.faces() {
|
||||
let path = match &face.source {
|
||||
Source::File(path) | Source::SharedFile(path, _) => path,
|
||||
// We never add binary sources to the database, so there
|
||||
// shouln't be any.
|
||||
Source::Binary(_) => continue,
|
||||
};
|
||||
|
||||
let info = db
|
||||
.with_face_data(face.id, FontInfo::new)
|
||||
.expect("database must contain this font");
|
||||
|
||||
if let Some(info) = info {
|
||||
self.book.push(info);
|
||||
self.fonts.push(FontSlot {
|
||||
path: path.clone(),
|
||||
index: face.index,
|
||||
font: OnceLock::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Embedded fonts have lowest priority.
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
self.add_embedded();
|
||||
}
|
||||
|
||||
/// Add fonts that are embedded in the binary.
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
fn add_embedded(&mut self) {
|
||||
for data in typst_assets::fonts() {
|
||||
let buffer = typst::foundations::Bytes::from_static(data);
|
||||
for (i, font) in Font::iter(buffer).enumerate() {
|
||||
self.book.push(font.info().clone());
|
||||
self.fonts.push(FontSlot {
|
||||
path: PathBuf::new(),
|
||||
index: i as u32,
|
||||
font: OnceLock::from(Some(font)),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -10,11 +10,12 @@ use typst::syntax::package::{
|
||||
};
|
||||
|
||||
use crate::args::InitCommand;
|
||||
use crate::package::PackageStorage;
|
||||
use crate::download::PrintDownload;
|
||||
use crate::package;
|
||||
|
||||
/// Execute an initialization command.
|
||||
pub fn init(command: &InitCommand) -> StrResult<()> {
|
||||
let package_storage = PackageStorage::from_args(&command.package_storage_args);
|
||||
let package_storage = package::storage(&command.package_storage_args);
|
||||
|
||||
// Parse the package specification. If the user didn't specify the version,
|
||||
// we try to figure it out automatically by downloading the package index
|
||||
@ -28,7 +29,8 @@ pub fn init(command: &InitCommand) -> StrResult<()> {
|
||||
})?;
|
||||
|
||||
// Find or download the package.
|
||||
let package_path = package_storage.prepare_package(&spec)?;
|
||||
let package_path =
|
||||
package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?;
|
||||
|
||||
// Parse the manifest.
|
||||
let manifest = parse_manifest(&package_path)?;
|
||||
|
@ -1,169 +1,13 @@
|
||||
use std::fs;
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use typst_kit::package::PackageStorage;
|
||||
|
||||
use crate::args::PackageStorageArgs;
|
||||
use codespan_reporting::term::{self, termcolor};
|
||||
use ecow::eco_format;
|
||||
use once_cell::sync::OnceCell;
|
||||
use termcolor::WriteColor;
|
||||
use typst::diag::{bail, PackageError, PackageResult, StrResult};
|
||||
use typst::syntax::package::{
|
||||
PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec,
|
||||
};
|
||||
use crate::download;
|
||||
|
||||
use crate::download::{download, download_with_progress};
|
||||
use crate::terminal;
|
||||
|
||||
const HOST: &str = "https://packages.typst.org";
|
||||
const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
|
||||
|
||||
/// Holds information about where packages should be stored.
|
||||
pub struct PackageStorage {
|
||||
pub package_cache_path: Option<PathBuf>,
|
||||
pub package_path: Option<PathBuf>,
|
||||
index: OnceCell<Vec<PackageInfo>>,
|
||||
}
|
||||
|
||||
impl PackageStorage {
|
||||
pub fn from_args(args: &PackageStorageArgs) -> Self {
|
||||
let package_cache_path = args.package_cache_path.clone().or_else(|| {
|
||||
dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR))
|
||||
});
|
||||
let package_path = args.package_path.clone().or_else(|| {
|
||||
dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR))
|
||||
});
|
||||
Self {
|
||||
package_cache_path,
|
||||
package_path,
|
||||
index: OnceCell::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Make a package available in the on-disk cache.
|
||||
pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult<PathBuf> {
|
||||
let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
|
||||
|
||||
if let Some(packages_dir) = &self.package_path {
|
||||
let dir = packages_dir.join(&subdir);
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(cache_dir) = &self.package_cache_path {
|
||||
let dir = cache_dir.join(&subdir);
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
|
||||
// Download from network if it doesn't exist yet.
|
||||
if spec.namespace == "preview" {
|
||||
self.download_package(spec, &dir)?;
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(PackageError::NotFound(spec.clone()))
|
||||
}
|
||||
|
||||
/// Try to determine the latest version of a package.
|
||||
pub fn determine_latest_version(
|
||||
&self,
|
||||
spec: &VersionlessPackageSpec,
|
||||
) -> StrResult<PackageVersion> {
|
||||
if spec.namespace == "preview" {
|
||||
// For `@preview`, download the package index and find the latest
|
||||
// version.
|
||||
self.download_index()?
|
||||
.iter()
|
||||
.filter(|package| package.name == spec.name)
|
||||
.map(|package| package.version)
|
||||
.max()
|
||||
.ok_or_else(|| eco_format!("failed to find package {spec}"))
|
||||
} else {
|
||||
// For other namespaces, search locally. We only search in the data
|
||||
// directory and not the cache directory, because the latter is not
|
||||
// intended for storage of local packages.
|
||||
let subdir = format!("{}/{}", spec.namespace, spec.name);
|
||||
self.package_path
|
||||
.iter()
|
||||
.flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
|
||||
.flatten()
|
||||
.filter_map(|entry| entry.ok())
|
||||
.map(|entry| entry.path())
|
||||
.filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
|
||||
.max()
|
||||
.ok_or_else(|| eco_format!("please specify the desired version"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PackageStorage {
|
||||
/// Download a package over the network.
|
||||
fn download_package(
|
||||
&self,
|
||||
spec: &PackageSpec,
|
||||
package_dir: &Path,
|
||||
) -> PackageResult<()> {
|
||||
// The `@preview` namespace is the only namespace that supports on-demand
|
||||
// fetching.
|
||||
assert_eq!(spec.namespace, "preview");
|
||||
|
||||
let url = format!("{HOST}/preview/{}-{}.tar.gz", spec.name, spec.version);
|
||||
|
||||
print_downloading(spec).unwrap();
|
||||
|
||||
let data = match download_with_progress(&url) {
|
||||
Ok(data) => data,
|
||||
Err(ureq::Error::Status(404, _)) => {
|
||||
if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
|
||||
return Err(PackageError::VersionNotFound(spec.clone(), version));
|
||||
} else {
|
||||
return Err(PackageError::NotFound(spec.clone()));
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))
|
||||
}
|
||||
};
|
||||
|
||||
let decompressed = flate2::read::GzDecoder::new(data.as_slice());
|
||||
tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
|
||||
fs::remove_dir_all(package_dir).ok();
|
||||
PackageError::MalformedArchive(Some(eco_format!("{err}")))
|
||||
})
|
||||
}
|
||||
|
||||
/// Download the `@preview` package index.
|
||||
///
|
||||
/// To avoid downloading the index multiple times, the result is cached.
|
||||
fn download_index(&self) -> StrResult<&Vec<PackageInfo>> {
|
||||
self.index.get_or_try_init(|| {
|
||||
let url = format!("{HOST}/preview/index.json");
|
||||
match download(&url) {
|
||||
Ok(response) => response
|
||||
.into_json()
|
||||
.map_err(|err| eco_format!("failed to parse package index: {err}")),
|
||||
Err(ureq::Error::Status(404, _)) => {
|
||||
bail!("failed to fetch package index (not found)")
|
||||
}
|
||||
Err(err) => bail!("failed to fetch package index ({err})"),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Print that a package downloading is happening.
|
||||
fn print_downloading(spec: &PackageSpec) -> io::Result<()> {
|
||||
let styles = term::Styles::default();
|
||||
|
||||
let mut out = terminal::out();
|
||||
out.set_color(&styles.header_help)?;
|
||||
write!(out, "downloading")?;
|
||||
|
||||
out.reset()?;
|
||||
writeln!(out, " {spec}")
|
||||
/// Returns a new package storage for the given args.
|
||||
pub fn storage(args: &PackageStorageArgs) -> PackageStorage {
|
||||
PackageStorage::new(
|
||||
args.package_cache_path.clone(),
|
||||
args.package_path.clone(),
|
||||
download::downloader(),
|
||||
)
|
||||
}
|
||||
|
@ -7,11 +7,12 @@ use semver::Version;
|
||||
use serde::Deserialize;
|
||||
use tempfile::NamedTempFile;
|
||||
use typst::diag::{bail, StrResult};
|
||||
use typst_kit::download::Downloader;
|
||||
use xz2::bufread::XzDecoder;
|
||||
use zip::ZipArchive;
|
||||
|
||||
use crate::args::UpdateCommand;
|
||||
use crate::download::{download, download_with_progress};
|
||||
use crate::download::{self, PrintDownload};
|
||||
|
||||
const TYPST_GITHUB_ORG: &str = "typst";
|
||||
const TYPST_REPO: &str = "typst";
|
||||
@ -68,13 +69,15 @@ pub fn update(command: &UpdateCommand) -> StrResult<()> {
|
||||
fs::copy(current_exe, &backup_path)
|
||||
.map_err(|err| eco_format!("failed to create backup ({err})"))?;
|
||||
|
||||
let release = Release::from_tag(command.version.as_ref())?;
|
||||
let downloader = download::downloader();
|
||||
|
||||
let release = Release::from_tag(command.version.as_ref(), &downloader)?;
|
||||
if !update_needed(&release)? && !command.force {
|
||||
eprintln!("Already up-to-date.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let binary_data = release.download_binary(needed_asset()?)?;
|
||||
let binary_data = release.download_binary(needed_asset()?, &downloader)?;
|
||||
let mut temp_exe = NamedTempFile::new()
|
||||
.map_err(|err| eco_format!("failed to create temporary file ({err})"))?;
|
||||
temp_exe
|
||||
@ -106,7 +109,10 @@ struct Release {
|
||||
impl Release {
|
||||
/// Download the target release, or latest if version is `None`, from the
|
||||
/// Typst repository.
|
||||
pub fn from_tag(tag: Option<&Version>) -> StrResult<Release> {
|
||||
pub fn from_tag(
|
||||
tag: Option<&Version>,
|
||||
downloader: &Downloader,
|
||||
) -> StrResult<Release> {
|
||||
let url = match tag {
|
||||
Some(tag) => format!(
|
||||
"https://api.github.com/repos/{TYPST_GITHUB_ORG}/{TYPST_REPO}/releases/tags/v{tag}"
|
||||
@ -116,7 +122,7 @@ impl Release {
|
||||
),
|
||||
};
|
||||
|
||||
match download(&url) {
|
||||
match downloader.download(&url) {
|
||||
Ok(response) => response.into_json().map_err(|err| {
|
||||
eco_format!("failed to parse release information ({err})")
|
||||
}),
|
||||
@ -130,15 +136,21 @@ impl Release {
|
||||
/// Download the binary from a given [`Release`] and select the
|
||||
/// corresponding asset for this target platform, returning the raw binary
|
||||
/// data.
|
||||
pub fn download_binary(&self, asset_name: &str) -> StrResult<Vec<u8>> {
|
||||
pub fn download_binary(
|
||||
&self,
|
||||
asset_name: &str,
|
||||
downloader: &Downloader,
|
||||
) -> StrResult<Vec<u8>> {
|
||||
let asset = self
|
||||
.assets
|
||||
.iter()
|
||||
.find(|a| a.name.starts_with(asset_name))
|
||||
.ok_or("could not find release for your target platform")?;
|
||||
|
||||
eprintln!("Downloading release ...");
|
||||
let data = match download_with_progress(&asset.browser_download_url) {
|
||||
let data = match downloader.download_with_progress(
|
||||
&asset.browser_download_url,
|
||||
&mut PrintDownload("release"),
|
||||
) {
|
||||
Ok(data) => data,
|
||||
Err(ureq::Error::Status(404, _)) => {
|
||||
bail!("asset not found (searched for {})", asset.name);
|
||||
|
@ -14,12 +14,14 @@ use typst::syntax::{FileId, Source, VirtualPath};
|
||||
use typst::text::{Font, FontBook};
|
||||
use typst::utils::LazyHash;
|
||||
use typst::{Library, World};
|
||||
use typst_kit::fonts::{FontSlot, Fonts};
|
||||
use typst_kit::package::PackageStorage;
|
||||
use typst_timing::{timed, TimingScope};
|
||||
|
||||
use crate::args::{Input, SharedArgs};
|
||||
use crate::compile::ExportCache;
|
||||
use crate::fonts::{FontSearcher, FontSlot};
|
||||
use crate::package::PackageStorage;
|
||||
use crate::download::PrintDownload;
|
||||
use crate::package;
|
||||
|
||||
/// Static `FileId` allocated for stdin.
|
||||
/// This is to ensure that a file is read in the correct way.
|
||||
@ -110,26 +112,24 @@ impl SystemWorld {
|
||||
Library::builder().with_inputs(inputs).build()
|
||||
};
|
||||
|
||||
let mut searcher = FontSearcher::new();
|
||||
searcher
|
||||
.search(&command.font_args.font_paths, command.font_args.ignore_system_fonts);
|
||||
let fonts = Fonts::searcher()
|
||||
.include_system_fonts(command.font_args.ignore_system_fonts)
|
||||
.search_with(&command.font_args.font_paths);
|
||||
|
||||
let now = match command.creation_timestamp {
|
||||
Some(time) => Now::Fixed(time),
|
||||
None => Now::System(OnceLock::new()),
|
||||
};
|
||||
|
||||
let package_storage = PackageStorage::from_args(&command.package_storage_args);
|
||||
|
||||
Ok(Self {
|
||||
workdir: std::env::current_dir().ok(),
|
||||
root,
|
||||
main,
|
||||
library: LazyHash::new(library),
|
||||
book: LazyHash::new(searcher.book),
|
||||
fonts: searcher.fonts,
|
||||
book: LazyHash::new(fonts.book),
|
||||
fonts: fonts.fonts,
|
||||
slots: Mutex::new(HashMap::new()),
|
||||
package_storage,
|
||||
package_storage: package::storage(&command.package_storage_args),
|
||||
now,
|
||||
export_cache: ExportCache::new(),
|
||||
})
|
||||
@ -378,7 +378,7 @@ fn system_path(
|
||||
let buf;
|
||||
let mut root = project_root;
|
||||
if let Some(spec) = id.package() {
|
||||
buf = package_storage.prepare_package(spec)?;
|
||||
buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?;
|
||||
root = &buf;
|
||||
}
|
||||
|
||||
|
57
crates/typst-kit/Cargo.toml
Normal file
57
crates/typst-kit/Cargo.toml
Normal file
@ -0,0 +1,57 @@
|
||||
[package]
|
||||
name = "typst-kit"
|
||||
description = "Common utilities for Typst tooling."
|
||||
version = { workspace = true }
|
||||
rust-version = { workspace = true }
|
||||
authors = { workspace = true }
|
||||
edition = { workspace = true }
|
||||
homepage = { workspace = true }
|
||||
repository = { workspace = true }
|
||||
license = { workspace = true }
|
||||
readme = { workspace = true }
|
||||
|
||||
[dependencies]
|
||||
typst = { workspace = true }
|
||||
typst-assets = { workspace = true, optional = true }
|
||||
typst-timing = { workspace = true }
|
||||
typst-utils = { workspace = true }
|
||||
ecow = { workspace = true }
|
||||
env_proxy = { workspace = true, optional = true }
|
||||
dirs = { workspace = true, optional = true }
|
||||
flate2 = { workspace = true, optional = true }
|
||||
fontdb = { workspace = true, optional = true }
|
||||
native-tls = { workspace = true, optional = true }
|
||||
once_cell = { workspace = true }
|
||||
tar = { workspace = true, optional = true }
|
||||
ureq = { workspace = true, optional = true }
|
||||
|
||||
# Explicitly depend on OpenSSL if applicable, so that we can add the
|
||||
# `openssl/vendored` feature to it if `vendor-openssl` is enabled.
|
||||
[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies]
|
||||
openssl = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = ["fonts", "packages"]
|
||||
|
||||
# Add font loading utilitites
|
||||
fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"]
|
||||
|
||||
# Add generic downloading utilities
|
||||
downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq"]
|
||||
|
||||
# Add package downloading utilities, implies `downloads`
|
||||
packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"]
|
||||
|
||||
# Embeds some fonts into the binary:
|
||||
# - For text: Linux Libertine, New Computer Modern
|
||||
# - For math: New Computer Modern Math
|
||||
# - For code: Deja Vu Sans Mono
|
||||
#
|
||||
# Implies `fonts`
|
||||
embed-fonts = ["fonts", "dep:typst-assets", "typst-assets/fonts"]
|
||||
|
||||
# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds.
|
||||
vendor-openssl = ["openssl/vendored"]
|
||||
|
||||
[lints]
|
||||
workspace = true
|
259
crates/typst-kit/src/download.rs
Normal file
259
crates/typst-kit/src/download.rs
Normal file
@ -0,0 +1,259 @@
|
||||
// Acknowledgement:
|
||||
// Closely modelled after rustup's `DownloadTracker`.
|
||||
// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
|
||||
|
||||
//! Helpers for making various web requests with status reporting. These are
|
||||
//! primarily used for communicating with package registries.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::fmt::Debug;
|
||||
use std::io::{self, ErrorKind, Read};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use ecow::EcoString;
|
||||
use native_tls::{Certificate, TlsConnector};
|
||||
use once_cell::sync::OnceCell;
|
||||
use ureq::Response;
|
||||
|
||||
/// Manages progress reporting for downloads.
|
||||
pub trait Progress {
|
||||
/// Invoked when a download is started.
|
||||
fn print_start(&mut self);
|
||||
|
||||
/// Invoked repeatedly while a download is ongoing.
|
||||
fn print_progress(&mut self, state: &DownloadState);
|
||||
|
||||
/// Invoked when a download is finished.
|
||||
fn print_finish(&mut self, state: &DownloadState);
|
||||
}
|
||||
|
||||
/// An implementation of [`Progress`] whth no-op reporting, i.e. reporting
|
||||
/// events are swallowed.
|
||||
pub struct ProgressSink;
|
||||
|
||||
impl Progress for ProgressSink {
|
||||
fn print_start(&mut self) {}
|
||||
fn print_progress(&mut self, _: &DownloadState) {}
|
||||
fn print_finish(&mut self, _: &DownloadState) {}
|
||||
}
|
||||
|
||||
/// The current state of an in progress or finished download.
|
||||
#[derive(Debug)]
|
||||
pub struct DownloadState {
|
||||
/// The expected amount of bytes to download, `None` if the response header
|
||||
/// was not set.
|
||||
pub content_len: Option<usize>,
|
||||
/// The total amount of downloaded bytes until now.
|
||||
pub total_downloaded: usize,
|
||||
/// A backlog of the amount of downloaded bytes each second.
|
||||
pub bytes_per_second: VecDeque<usize>,
|
||||
/// The download starting instant.
|
||||
pub start_time: Instant,
|
||||
}
|
||||
|
||||
/// A minimal https client for downloading various resources.
|
||||
pub struct Downloader {
|
||||
user_agent: EcoString,
|
||||
cert_path: Option<PathBuf>,
|
||||
cert: OnceCell<Certificate>,
|
||||
}
|
||||
|
||||
impl Downloader {
|
||||
/// Crates a new downloader with the given user agent and no certificate.
|
||||
pub fn new(user_agent: impl Into<EcoString>) -> Self {
|
||||
Self {
|
||||
user_agent: user_agent.into(),
|
||||
cert_path: None,
|
||||
cert: OnceCell::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Crates a new downloader with the given user agent and certificate path.
|
||||
///
|
||||
/// If the certificate cannot be read it is set to `None`.
|
||||
pub fn with_path(user_agent: impl Into<EcoString>, cert_path: PathBuf) -> Self {
|
||||
Self {
|
||||
user_agent: user_agent.into(),
|
||||
cert_path: Some(cert_path),
|
||||
cert: OnceCell::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Crates a new downloader with the given user agent and certificate.
|
||||
pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self {
|
||||
Self {
|
||||
user_agent: user_agent.into(),
|
||||
cert_path: None,
|
||||
cert: OnceCell::with_value(cert),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the certificate this client is using, if a custom certificate
|
||||
/// is used it is loaded on first access.
|
||||
///
|
||||
/// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
|
||||
/// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
|
||||
/// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
|
||||
pub fn cert(&self) -> Option<io::Result<&Certificate>> {
|
||||
self.cert_path.as_ref().map(|path| {
|
||||
self.cert.get_or_try_init(|| {
|
||||
let pem = std::fs::read(path)?;
|
||||
Certificate::from_pem(&pem).map_err(io::Error::other)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Download binary data from the given url.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
|
||||
let mut builder = ureq::AgentBuilder::new();
|
||||
let mut tls = TlsConnector::builder();
|
||||
|
||||
// Set user agent.
|
||||
builder = builder.user_agent(&self.user_agent);
|
||||
|
||||
// Get the network proxy config from the environment and apply it.
|
||||
if let Some(proxy) = env_proxy::for_url_str(url)
|
||||
.to_url()
|
||||
.and_then(|url| ureq::Proxy::new(url).ok())
|
||||
{
|
||||
builder = builder.proxy(proxy);
|
||||
}
|
||||
|
||||
// Apply a custom CA certificate if present.
|
||||
if let Some(cert) = self.cert() {
|
||||
tls.add_root_certificate(cert?.clone());
|
||||
}
|
||||
|
||||
// Configure native TLS.
|
||||
let connector =
|
||||
tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
|
||||
builder = builder.tls_connector(Arc::new(connector));
|
||||
|
||||
builder.build().get(url).call()
|
||||
}
|
||||
|
||||
/// Download binary data from the given url and report its progress.
|
||||
#[allow(clippy::result_large_err)]
|
||||
pub fn download_with_progress(
|
||||
&self,
|
||||
url: &str,
|
||||
progress: &mut dyn Progress,
|
||||
) -> Result<Vec<u8>, ureq::Error> {
|
||||
progress.print_start();
|
||||
let response = self.download(url)?;
|
||||
Ok(RemoteReader::from_response(response, progress).download()?)
|
||||
}
|
||||
}
|
||||
|
||||
impl Debug for Downloader {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Downloader")
|
||||
.field("user_agent", &self.user_agent)
|
||||
.field("cert_path", &self.cert_path)
|
||||
.field(
|
||||
"cert",
|
||||
&self
|
||||
.cert
|
||||
.get()
|
||||
.map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))),
|
||||
)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep track of this many download speed samples.
|
||||
const SAMPLES: usize = 5;
|
||||
|
||||
/// A wrapper around [`ureq::Response`] that reads the response body in chunks
|
||||
/// over a websocket and reports its progress.
|
||||
struct RemoteReader<'p> {
|
||||
/// The reader returned by the ureq::Response.
|
||||
reader: Box<dyn Read + Send + Sync + 'static>,
|
||||
/// The download state, holding download metadata for progress reporting.
|
||||
state: DownloadState,
|
||||
/// The instant at which progress was last reported.
|
||||
last_progress: Option<Instant>,
|
||||
/// A trait object used to report download progress.
|
||||
progress: &'p mut dyn Progress,
|
||||
}
|
||||
|
||||
impl<'p> RemoteReader<'p> {
|
||||
/// Wraps a [`ureq::Response`] and prepares it for downloading.
|
||||
///
|
||||
/// The 'Content-Length' header is used as a size hint for read
|
||||
/// optimization, if present.
|
||||
fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self {
|
||||
let content_len: Option<usize> = response
|
||||
.header("Content-Length")
|
||||
.and_then(|header| header.parse().ok());
|
||||
|
||||
Self {
|
||||
reader: response.into_reader(),
|
||||
last_progress: None,
|
||||
state: DownloadState {
|
||||
content_len,
|
||||
total_downloaded: 0,
|
||||
bytes_per_second: VecDeque::with_capacity(SAMPLES),
|
||||
start_time: Instant::now(),
|
||||
},
|
||||
progress,
|
||||
}
|
||||
}
|
||||
|
||||
/// Download the body's content as raw bytes while reporting download
|
||||
/// progress.
|
||||
fn download(mut self) -> io::Result<Vec<u8>> {
|
||||
let mut buffer = vec![0; 8192];
|
||||
let mut data = match self.state.content_len {
|
||||
Some(content_len) => Vec::with_capacity(content_len),
|
||||
None => Vec::with_capacity(8192),
|
||||
};
|
||||
|
||||
let mut downloaded_this_sec = 0;
|
||||
loop {
|
||||
let read = match self.reader.read(&mut buffer) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => n,
|
||||
// If the data is not yet ready but will be available eventually
|
||||
// keep trying until we either get an actual error, receive data
|
||||
// or an Ok(0).
|
||||
Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
data.extend(&buffer[..read]);
|
||||
|
||||
let last_printed = match self.last_progress {
|
||||
Some(prev) => prev,
|
||||
None => {
|
||||
let current_time = Instant::now();
|
||||
self.last_progress = Some(current_time);
|
||||
current_time
|
||||
}
|
||||
};
|
||||
let elapsed = Instant::now().saturating_duration_since(last_printed);
|
||||
|
||||
downloaded_this_sec += read;
|
||||
self.state.total_downloaded += read;
|
||||
|
||||
if elapsed >= Duration::from_secs(1) {
|
||||
if self.state.bytes_per_second.len() == SAMPLES {
|
||||
self.state.bytes_per_second.pop_back();
|
||||
}
|
||||
|
||||
self.state.bytes_per_second.push_front(downloaded_this_sec);
|
||||
downloaded_this_sec = 0;
|
||||
|
||||
self.progress.print_progress(&self.state);
|
||||
self.last_progress = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
self.progress.print_finish(&self.state);
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
}
|
216
crates/typst-kit/src/fonts.rs
Normal file
216
crates/typst-kit/src/fonts.rs
Normal file
@ -0,0 +1,216 @@
|
||||
//! Default implementation for searching local and system installed fonts as
|
||||
//! well as loading embedded default fonts.
|
||||
//!
|
||||
//! # Embedded fonts
|
||||
//! The following fonts are available as embedded fonts via the `embed-fonts`
|
||||
//! feature flag:
|
||||
//! - For text: Linux Libertine, New Computer Modern
|
||||
//! - For math: New Computer Modern Math
|
||||
//! - For code: Deja Vu Sans Mono
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::OnceLock;
|
||||
use std::{fs, path::Path};
|
||||
|
||||
use fontdb::{Database, Source};
|
||||
use typst::text::{Font, FontBook, FontInfo};
|
||||
use typst_timing::TimingScope;
|
||||
|
||||
/// Holds details about the location of a font and lazily the font itself.
|
||||
#[derive(Debug)]
|
||||
pub struct FontSlot {
|
||||
/// The path at which the font can be found on the system.
|
||||
path: Option<PathBuf>,
|
||||
/// The index of the font in its collection. Zero if the path does not point
|
||||
/// to a collection.
|
||||
index: u32,
|
||||
/// The lazily loaded font.
|
||||
font: OnceLock<Option<Font>>,
|
||||
}
|
||||
|
||||
impl FontSlot {
|
||||
/// Returns the path at which the font can be found on the system, or `None`
|
||||
/// if the font was embedded.
|
||||
pub fn path(&self) -> Option<&Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
|
||||
/// Returns the index of the font in its collection. Zero if the path does
|
||||
/// not point to a collection.
|
||||
pub fn index(&self) -> u32 {
|
||||
self.index
|
||||
}
|
||||
|
||||
/// Get the font for this slot. This loads the font into memory on first
|
||||
/// access.
|
||||
pub fn get(&self) -> Option<Font> {
|
||||
self.font
|
||||
.get_or_init(|| {
|
||||
let _scope = TimingScope::new("load font", None);
|
||||
let data = fs::read(
|
||||
self.path
|
||||
.as_ref()
|
||||
.expect("`path` is not `None` if `font` is uninitialized"),
|
||||
)
|
||||
.ok()?
|
||||
.into();
|
||||
Font::new(data, self.index)
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// The result of a font search, created by calling [`FontSearcher::search`].
|
||||
#[derive(Debug)]
|
||||
pub struct Fonts {
|
||||
/// Metadata about all discovered fonts.
|
||||
pub book: FontBook,
|
||||
/// Slots that the fonts are loaded into.
|
||||
pub fonts: Vec<FontSlot>,
|
||||
}
|
||||
|
||||
impl Fonts {
|
||||
/// Creates a new font searcer with the default settings.
|
||||
pub fn searcher() -> FontSearcher {
|
||||
FontSearcher::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Searches for fonts.
|
||||
///
|
||||
/// Fonts are added in the following order (descending priority):
|
||||
/// 1. Font directories
|
||||
/// 2. System fonts (if included & enabled)
|
||||
/// 3. Embedded fonts (if enabled)
|
||||
#[derive(Debug)]
|
||||
pub struct FontSearcher {
|
||||
db: Database,
|
||||
include_system_fonts: bool,
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
include_embedded_fonts: bool,
|
||||
book: FontBook,
|
||||
fonts: Vec<FontSlot>,
|
||||
}
|
||||
|
||||
impl FontSearcher {
|
||||
/// Create a new, empty system searcher. The searcher is created with the
|
||||
/// default configuration, it will include embedded fonts and system fonts.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
db: Database::new(),
|
||||
include_system_fonts: true,
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
include_embedded_fonts: true,
|
||||
book: FontBook::new(),
|
||||
fonts: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether to search for and load system fonts, defaults to `true`.
|
||||
pub fn include_system_fonts(&mut self, value: bool) -> &mut Self {
|
||||
self.include_system_fonts = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Whether to load embedded fonts, defaults to `true`.
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
pub fn include_embedded_fonts(&mut self, value: bool) -> &mut Self {
|
||||
self.include_embedded_fonts = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Start searching for and loading fonts. To additionally load fonts
|
||||
/// from specific directories, use [`search_with`][Self::search_with].
|
||||
///
|
||||
/// # Examples
|
||||
/// ```no_run
|
||||
/// # use typst_kit::fonts::FontSearcher;
|
||||
/// let fonts = FontSearcher::new()
|
||||
/// .include_system_fonts(true)
|
||||
/// .search();
|
||||
/// ```
|
||||
pub fn search(&mut self) -> Fonts {
|
||||
self.search_with::<_, &str>([])
|
||||
}
|
||||
|
||||
/// Start searching for and loading fonts, with additional directories.
|
||||
///
|
||||
/// # Examples
|
||||
/// ```no_run
|
||||
/// # use typst_kit::fonts::FontSearcher;
|
||||
/// let fonts = FontSearcher::new()
|
||||
/// .include_system_fonts(true)
|
||||
/// .search_with(["./assets/fonts/"]);
|
||||
/// ```
|
||||
pub fn search_with<I, P>(&mut self, font_dirs: I) -> Fonts
|
||||
where
|
||||
I: IntoIterator<Item = P>,
|
||||
P: AsRef<Path>,
|
||||
{
|
||||
// Font paths have highest priority.
|
||||
for path in font_dirs {
|
||||
self.db.load_fonts_dir(path);
|
||||
}
|
||||
|
||||
if self.include_system_fonts {
|
||||
// System fonts have second priority.
|
||||
self.db.load_system_fonts();
|
||||
}
|
||||
|
||||
for face in self.db.faces() {
|
||||
let path = match &face.source {
|
||||
Source::File(path) | Source::SharedFile(path, _) => path,
|
||||
// We never add binary sources to the database, so there
|
||||
// shouln't be any.
|
||||
Source::Binary(_) => continue,
|
||||
};
|
||||
|
||||
let info = self
|
||||
.db
|
||||
.with_face_data(face.id, FontInfo::new)
|
||||
.expect("database must contain this font");
|
||||
|
||||
if let Some(info) = info {
|
||||
self.book.push(info);
|
||||
self.fonts.push(FontSlot {
|
||||
path: Some(path.clone()),
|
||||
index: face.index,
|
||||
font: OnceLock::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Embedded fonts have lowest priority.
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
if self.include_embedded_fonts {
|
||||
self.add_embedded();
|
||||
}
|
||||
|
||||
Fonts {
|
||||
book: std::mem::take(&mut self.book),
|
||||
fonts: std::mem::take(&mut self.fonts),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add fonts that are embedded in the binary.
|
||||
#[cfg(feature = "embed-fonts")]
|
||||
fn add_embedded(&mut self) {
|
||||
for data in typst_assets::fonts() {
|
||||
let buffer = typst::foundations::Bytes::from_static(data);
|
||||
for (i, font) in Font::iter(buffer).enumerate() {
|
||||
self.book.push(font.info().clone());
|
||||
self.fonts.push(FontSlot {
|
||||
path: None,
|
||||
index: i as u32,
|
||||
font: OnceLock::from(Some(font)),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FontSearcher {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
27
crates/typst-kit/src/lib.rs
Normal file
27
crates/typst-kit/src/lib.rs
Normal file
@ -0,0 +1,27 @@
|
||||
//! Typst-kit contains various default implementations of functionality used in
|
||||
//! typst-cli. It is intended as a single source of truth for things like font
|
||||
//! searching, package downloads and more. Each component of typst-kit is
|
||||
//! optional, but enabled by default.
|
||||
//!
|
||||
//! # Components
|
||||
//! - [fonts] contains a default implementation for searching local and system
|
||||
//! installed fonts. It is enabled by the `fonts` feature flag, additionally
|
||||
//! the `embed-fonts` feature can be used to embed the Typst default fonts.
|
||||
//! - For text: Linux Libertine, New Computer Modern
|
||||
//! - For math: New Computer Modern Math
|
||||
//! - For code: Deja Vu Sans Mono
|
||||
//! - [download] contains functionality for making simple web requests with
|
||||
//! status reporting, useful for downloading packages from package registires.
|
||||
//! It is enabled by the `downloads` feature flag, additionally the
|
||||
//! `vendor-openssl` can be used on operating systems other than macOS and
|
||||
//! Windows to vendor OpenSSL when building.
|
||||
//! - [package] contains package storage and downloading functionality based on
|
||||
//! [download]. It is enabled by the `packages` feature flag and implies the
|
||||
//! `downloads` feature flag.
|
||||
|
||||
#[cfg(feature = "downloads")]
|
||||
pub mod download;
|
||||
#[cfg(feature = "fonts")]
|
||||
pub mod fonts;
|
||||
#[cfg(feature = "packages")]
|
||||
pub mod package;
|
179
crates/typst-kit/src/package.rs
Normal file
179
crates/typst-kit/src/package.rs
Normal file
@ -0,0 +1,179 @@
|
||||
//! Download and unpack packages and package indices.
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use ecow::eco_format;
|
||||
use once_cell::sync::OnceCell;
|
||||
use typst::diag::{bail, PackageError, PackageResult, StrResult};
|
||||
use typst::syntax::package::{
|
||||
PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec,
|
||||
};
|
||||
|
||||
use crate::download::{Downloader, Progress};
|
||||
|
||||
/// The default Typst registry.
|
||||
pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
|
||||
|
||||
/// The default packages sub directory within the package and package cache paths.
|
||||
pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
|
||||
|
||||
/// Holds information about where packages should be stored and downloads them
|
||||
/// on demand, if possible.
|
||||
#[derive(Debug)]
|
||||
pub struct PackageStorage {
|
||||
/// The path at which non-local packages should be stored when downloaded.
|
||||
package_cache_path: Option<PathBuf>,
|
||||
/// The path at which local packages are stored.
|
||||
package_path: Option<PathBuf>,
|
||||
/// The downloader used for fetching the index and packages.
|
||||
downloader: Downloader,
|
||||
/// The cached index of the preview namespace.
|
||||
index: OnceCell<Vec<PackageInfo>>,
|
||||
}
|
||||
|
||||
impl PackageStorage {
|
||||
/// Creates a new package storage for the given package paths. Falls back to
|
||||
/// the recommended XDG directories if they are `None`.
|
||||
pub fn new(
|
||||
package_cache_path: Option<PathBuf>,
|
||||
package_path: Option<PathBuf>,
|
||||
downloader: Downloader,
|
||||
) -> Self {
|
||||
Self {
|
||||
package_cache_path: package_cache_path.or_else(|| {
|
||||
dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR))
|
||||
}),
|
||||
package_path: package_path.or_else(|| {
|
||||
dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR))
|
||||
}),
|
||||
downloader,
|
||||
index: OnceCell::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a the path at which non-local packages should be stored when
|
||||
/// downloaded.
|
||||
pub fn package_cache_path(&self) -> Option<&Path> {
|
||||
self.package_cache_path.as_deref()
|
||||
}
|
||||
|
||||
/// Returns a the path at which local packages are stored.
|
||||
pub fn package_path(&self) -> Option<&Path> {
|
||||
self.package_path.as_deref()
|
||||
}
|
||||
|
||||
/// Make a package available in the on-disk.
|
||||
pub fn prepare_package(
|
||||
&self,
|
||||
spec: &PackageSpec,
|
||||
progress: &mut dyn Progress,
|
||||
) -> PackageResult<PathBuf> {
|
||||
let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
|
||||
|
||||
if let Some(packages_dir) = &self.package_path {
|
||||
let dir = packages_dir.join(&subdir);
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(cache_dir) = &self.package_cache_path {
|
||||
let dir = cache_dir.join(&subdir);
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
|
||||
// Download from network if it doesn't exist yet.
|
||||
self.download_package(spec, &dir, progress)?;
|
||||
if dir.exists() {
|
||||
return Ok(dir);
|
||||
}
|
||||
}
|
||||
|
||||
Err(PackageError::NotFound(spec.clone()))
|
||||
}
|
||||
|
||||
/// Try to determine the latest version of a package.
|
||||
pub fn determine_latest_version(
|
||||
&self,
|
||||
spec: &VersionlessPackageSpec,
|
||||
) -> StrResult<PackageVersion> {
|
||||
if spec.namespace == "preview" {
|
||||
// For `@preview`, download the package index and find the latest
|
||||
// version.
|
||||
self.download_index()?
|
||||
.iter()
|
||||
.filter(|package| package.name == spec.name)
|
||||
.map(|package| package.version)
|
||||
.max()
|
||||
.ok_or_else(|| eco_format!("failed to find package {spec}"))
|
||||
} else {
|
||||
// For other namespaces, search locally. We only search in the data
|
||||
// directory and not the cache directory, because the latter is not
|
||||
// intended for storage of local packages.
|
||||
let subdir = format!("{}/{}", spec.namespace, spec.name);
|
||||
self.package_path
|
||||
.iter()
|
||||
.flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
|
||||
.flatten()
|
||||
.filter_map(|entry| entry.ok())
|
||||
.map(|entry| entry.path())
|
||||
.filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
|
||||
.max()
|
||||
.ok_or_else(|| eco_format!("please specify the desired version"))
|
||||
}
|
||||
}
|
||||
|
||||
/// Download the package index. The result of this is cached for efficiency.
|
||||
pub fn download_index(&self) -> StrResult<&Vec<PackageInfo>> {
|
||||
self.index.get_or_try_init(|| {
|
||||
let url = format!("{DEFAULT_REGISTRY}/preview/index.json");
|
||||
match self.downloader.download(&url) {
|
||||
Ok(response) => response
|
||||
.into_json()
|
||||
.map_err(|err| eco_format!("failed to parse package index: {err}")),
|
||||
Err(ureq::Error::Status(404, _)) => {
|
||||
bail!("failed to fetch package index (not found)")
|
||||
}
|
||||
Err(err) => bail!("failed to fetch package index ({err})"),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Download a package over the network.
|
||||
///
|
||||
/// # Panics
|
||||
/// Panics if the package spec namespace isn't `preview`.
|
||||
pub fn download_package(
|
||||
&self,
|
||||
spec: &PackageSpec,
|
||||
package_dir: &Path,
|
||||
progress: &mut dyn Progress,
|
||||
) -> PackageResult<()> {
|
||||
assert_eq!(spec.namespace, "preview");
|
||||
|
||||
let url =
|
||||
format!("{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz", spec.name, spec.version);
|
||||
|
||||
let data = match self.downloader.download_with_progress(&url, progress) {
|
||||
Ok(data) => data,
|
||||
Err(ureq::Error::Status(404, _)) => {
|
||||
if let Ok(version) = self.determine_latest_version(&spec.versionless()) {
|
||||
return Err(PackageError::VersionNotFound(spec.clone(), version));
|
||||
} else {
|
||||
return Err(PackageError::NotFound(spec.clone()));
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(PackageError::NetworkFailed(Some(eco_format!("{err}"))))
|
||||
}
|
||||
};
|
||||
|
||||
let decompressed = flate2::read::GzDecoder::new(data.as_slice());
|
||||
tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| {
|
||||
fs::remove_dir_all(package_dir).ok();
|
||||
PackageError::MalformedArchive(Some(eco_format!("{err}")))
|
||||
})
|
||||
}
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user