diff --git a/Cargo.lock b/Cargo.lock index 9d5e9283a..d43899f70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2670,15 +2670,11 @@ dependencies = [ "comemo", "dirs", "ecow", - "env_proxy", - "flate2", - "fontdb", "fs_extra", "native-tls", "notify", "once_cell", "open", - "openssl", "parking_lot", "pathdiff", "rayon", @@ -2694,6 +2690,7 @@ dependencies = [ "toml", "typst", "typst-assets", + "typst-kit", "typst-macros", "typst-pdf", "typst-render", @@ -2760,6 +2757,26 @@ dependencies = [ "unscanny", ] +[[package]] +name = "typst-kit" +version = "0.11.0" +dependencies = [ + "dirs", + "ecow", + "env_proxy", + "flate2", + "fontdb", + "native-tls", + "once_cell", + "openssl", + "tar", + "typst", + "typst-assets", + "typst-timing", + "typst-utils", + "ureq", +] + [[package]] name = "typst-macros" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index bfd3b1401..18b670f0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ readme = "README.md" typst = { path = "crates/typst", version = "0.11.0" } typst-cli = { path = "crates/typst-cli", version = "0.11.0" } typst-ide = { path = "crates/typst-ide", version = "0.11.0" } +typst-kit = { path = "crates/typst-kit", version = "0.11.0" } typst-macros = { path = "crates/typst-macros", version = "0.11.0" } typst-pdf = { path = "crates/typst-pdf", version = "0.11.0" } typst-render = { path = "crates/typst-render", version = "0.11.0" } diff --git a/crates/typst-cli/Cargo.toml b/crates/typst-cli/Cargo.toml index 7ada123cb..31f19f394 100644 --- a/crates/typst-cli/Cargo.toml +++ b/crates/typst-cli/Cargo.toml @@ -20,6 +20,7 @@ doc = false [dependencies] typst = { workspace = true } typst-assets = { workspace = true, features = ["fonts"] } +typst-kit = { workspace = true } typst-macros = { workspace = true } typst-pdf = { workspace = true } typst-render = { workspace = true } @@ -31,9 +32,6 @@ codespan-reporting = { workspace = true } comemo = { workspace = true } dirs = { workspace = true } ecow = { workspace = true } -env_proxy = { workspace = true } -flate2 = { workspace = true } -fontdb = { workspace = true, features = ["memmap", "fontconfig"] } fs_extra = { workspace = true } native-tls = { workspace = true } notify = { workspace = true } @@ -56,11 +54,6 @@ ureq = { workspace = true } xz2 = { workspace = true, optional = true } zip = { workspace = true, optional = true } -# Explicitly depend on OpenSSL if applicable, so that we can add the -# `openssl/vendored` feature to it if `vendor-openssl` is enabled. -[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies] -openssl = { workspace = true } - [build-dependencies] chrono = { workspace = true } clap = { workspace = true, features = ["string"] } @@ -71,17 +64,14 @@ semver = { workspace = true } [features] default = ["embed-fonts"] -# Embeds some fonts into the binary: -# - For text: Linux Libertine, New Computer Modern -# - For math: New Computer Modern Math -# - For code: Deja Vu Sans Mono -embed-fonts = [] +# Embeds some fonts into the binary, see typst-kit +embed-fonts = ["typst-kit/embed-fonts"] # Permits the CLI to update itself without a package manager. self-update = ["dep:self-replace", "dep:xz2", "dep:zip"] # Whether to vendor OpenSSL. Not applicable to Windows and macOS builds. -vendor-openssl = ["openssl/vendored"] +vendor-openssl = ["typst-kit/vendor-openssl"] [lints] workspace = true diff --git a/crates/typst-cli/src/download.rs b/crates/typst-cli/src/download.rs index 63a2e4163..8082fa52b 100644 --- a/crates/typst-cli/src/download.rs +++ b/crates/typst-cli/src/download.rs @@ -1,205 +1,90 @@ -// Acknowledgement: -// Closely modelled after rustup's `DownloadTracker`. -// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs - -use std::collections::VecDeque; -use std::io::{self, ErrorKind, Read, Write}; -use std::sync::Arc; +use std::fmt::Display; +use std::io; +use std::io::Write; use std::time::{Duration, Instant}; -use native_tls::{Certificate, TlsConnector}; -use once_cell::sync::OnceCell; -use ureq::Response; +use codespan_reporting::term; +use codespan_reporting::term::termcolor::WriteColor; +use typst_kit::download::{DownloadState, Downloader, Progress}; -use crate::terminal; +use crate::terminal::{self, TermOut}; +use crate::ARGS; -/// Keep track of this many download speed samples. -const SPEED_SAMPLES: usize = 5; +/// Prints download progress by writing `downloading {0}` followed by repeatedly +/// updating the last terminal line. +pub struct PrintDownload(pub T); -/// Load a certificate from the file system if the `--cert` argument or -/// `TYPST_CERT` environment variable is present. The certificate is cached for -/// efficiency. -/// -/// - Returns `None` if `--cert` and `TYPST_CERT` are not set. -/// - Returns `Some(Ok(cert))` if the certificate was loaded successfully. -/// - Returns `Some(Err(err))` if an error occurred while loading the certificate. -fn cert() -> Option> { - static CERT: OnceCell = OnceCell::new(); - crate::ARGS.cert.as_ref().map(|path| { - CERT.get_or_try_init(|| { - let pem = std::fs::read(path)?; - Certificate::from_pem(&pem).map_err(io::Error::other) - }) - }) -} +impl Progress for PrintDownload { + fn print_start(&mut self) { + // Print that a package downloading is happening. + let styles = term::Styles::default(); -/// Download binary data and display its progress. -#[allow(clippy::result_large_err)] -pub fn download_with_progress(url: &str) -> Result, ureq::Error> { - let response = download(url)?; - Ok(RemoteReader::from_response(response).download()?) -} + let mut out = terminal::out(); + let _ = out.set_color(&styles.header_help); + let _ = write!(out, "downloading"); -/// Download from a URL. -#[allow(clippy::result_large_err)] -pub fn download(url: &str) -> Result { - let mut builder = ureq::AgentBuilder::new(); - let mut tls = TlsConnector::builder(); - - // Set user agent. - builder = builder.user_agent(concat!("typst/", env!("CARGO_PKG_VERSION"))); - - // Get the network proxy config from the environment and apply it. - if let Some(proxy) = env_proxy::for_url_str(url) - .to_url() - .and_then(|url| ureq::Proxy::new(url).ok()) - { - builder = builder.proxy(proxy); + let _ = out.reset(); + let _ = writeln!(out, " {}", self.0); } - // Apply a custom CA certificate if present. - if let Some(cert) = cert() { - tls.add_root_certificate(cert?.clone()); + fn print_progress(&mut self, state: &DownloadState) { + let mut out = terminal::out(); + let _ = out.clear_last_line(); + let _ = display_download_progress(&mut out, state); } - // Configure native TLS. - let connector = - tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?; - builder = builder.tls_connector(Arc::new(connector)); - - builder.build().get(url).call() + fn print_finish(&mut self, state: &DownloadState) { + let mut out = terminal::out(); + let _ = display_download_progress(&mut out, state); + let _ = writeln!(out); + } } -/// A wrapper around [`ureq::Response`] that reads the response body in chunks -/// over a websocket and displays statistics about its progress. -/// -/// Downloads will _never_ fail due to statistics failing to print, print errors -/// are silently ignored. -struct RemoteReader { - reader: Box, - content_len: Option, - total_downloaded: usize, - downloaded_this_sec: usize, - downloaded_last_few_secs: VecDeque, - start_time: Instant, - last_print: Option, +/// Returns a new downloader. +pub fn downloader() -> Downloader { + let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION")); + match ARGS.cert.clone() { + Some(cert) => Downloader::with_path(user_agent, cert), + None => Downloader::new(user_agent), + } } -impl RemoteReader { - /// Wraps a [`ureq::Response`] and prepares it for downloading. - /// - /// The 'Content-Length' header is used as a size hint for read - /// optimization, if present. - pub fn from_response(response: Response) -> Self { - let content_len: Option = response - .header("Content-Length") - .and_then(|header| header.parse().ok()); +/// Compile and format several download statistics and make and attempt at +/// displaying them on standard error. +pub fn display_download_progress( + out: &mut TermOut, + state: &DownloadState, +) -> io::Result<()> { + let sum: usize = state.bytes_per_second.iter().sum(); + let len = state.bytes_per_second.len(); + let speed = if len > 0 { sum / len } else { state.content_len.unwrap_or(0) }; - Self { - reader: response.into_reader(), - content_len, - total_downloaded: 0, - downloaded_this_sec: 0, - downloaded_last_few_secs: VecDeque::with_capacity(SPEED_SAMPLES), - start_time: Instant::now(), - last_print: None, + let total_downloaded = as_bytes_unit(state.total_downloaded); + let speed_h = as_throughput_unit(speed); + let elapsed = time_suffix(Instant::now().saturating_duration_since(state.start_time)); + + match state.content_len { + Some(content_len) => { + let percent = (state.total_downloaded as f64 / content_len as f64) * 100.; + let remaining = content_len - state.total_downloaded; + + let download_size = as_bytes_unit(content_len); + let eta = time_suffix(Duration::from_secs(if speed == 0 { + 0 + } else { + (remaining / speed) as u64 + })); + writeln!( + out, + "{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}", + )?; } - } - - /// Download the bodies content as raw bytes while attempting to print - /// download statistics to standard error. Download progress gets displayed - /// and updated every second. - /// - /// These statistics will never prevent a download from completing, errors - /// are silently ignored. - pub fn download(mut self) -> io::Result> { - let mut buffer = vec![0; 8192]; - let mut data = match self.content_len { - Some(content_len) => Vec::with_capacity(content_len), - None => Vec::with_capacity(8192), - }; - - loop { - let read = match self.reader.read(&mut buffer) { - Ok(0) => break, - Ok(n) => n, - // If the data is not yet ready but will be available eventually - // keep trying until we either get an actual error, receive data - // or an Ok(0). - Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - - data.extend(&buffer[..read]); - - let last_printed = match self.last_print { - Some(prev) => prev, - None => { - let current_time = Instant::now(); - self.last_print = Some(current_time); - current_time - } - }; - let elapsed = Instant::now().saturating_duration_since(last_printed); - - self.total_downloaded += read; - self.downloaded_this_sec += read; - - if elapsed >= Duration::from_secs(1) { - if self.downloaded_last_few_secs.len() == SPEED_SAMPLES { - self.downloaded_last_few_secs.pop_back(); - } - - self.downloaded_last_few_secs.push_front(self.downloaded_this_sec); - self.downloaded_this_sec = 0; - - terminal::out().clear_last_line()?; - self.display()?; - self.last_print = Some(Instant::now()); - } - } - - self.display()?; - writeln!(&mut terminal::out())?; - - Ok(data) - } - - /// Compile and format several download statistics and make an attempt at - /// displaying them on standard error. - fn display(&mut self) -> io::Result<()> { - let sum: usize = self.downloaded_last_few_secs.iter().sum(); - let len = self.downloaded_last_few_secs.len(); - let speed = if len > 0 { sum / len } else { self.content_len.unwrap_or(0) }; - - let total_downloaded = as_bytes_unit(self.total_downloaded); - let speed_h = as_throughput_unit(speed); - let elapsed = - time_suffix(Instant::now().saturating_duration_since(self.start_time)); - - match self.content_len { - Some(content_len) => { - let percent = (self.total_downloaded as f64 / content_len as f64) * 100.; - let remaining = content_len - self.total_downloaded; - - let download_size = as_bytes_unit(content_len); - let eta = time_suffix(Duration::from_secs(if speed == 0 { - 0 - } else { - (remaining / speed) as u64 - })); - writeln!( - terminal::out(), - "{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}", - )?; - } - None => writeln!( - terminal::out(), - "Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}", - )?, - }; - Ok(()) - } + None => writeln!( + out, + "Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}", + )?, + }; + Ok(()) } /// Append a unit-of-time suffix. diff --git a/crates/typst-cli/src/fonts.rs b/crates/typst-cli/src/fonts.rs index de9d1fc1b..f5aa9826a 100644 --- a/crates/typst-cli/src/fonts.rs +++ b/crates/typst-cli/src/fonts.rs @@ -1,20 +1,16 @@ -use std::fs; -use std::path::PathBuf; -use std::sync::OnceLock; - -use fontdb::{Database, Source}; use typst::diag::StrResult; -use typst::text::{Font, FontBook, FontInfo, FontVariant}; -use typst_timing::TimingScope; +use typst::text::FontVariant; +use typst_kit::fonts::Fonts; use crate::args::FontsCommand; /// Execute a font listing command. pub fn fonts(command: &FontsCommand) -> StrResult<()> { - let mut searcher = FontSearcher::new(); - searcher.search(&command.font_args.font_paths, command.font_args.ignore_system_fonts); + let fonts = Fonts::searcher() + .include_system_fonts(!command.font_args.ignore_system_fonts) + .search_with(&command.font_args.font_paths); - for (name, infos) in searcher.book.families() { + for (name, infos) in fonts.book.families() { println!("{name}"); if command.variants { for info in infos { @@ -26,99 +22,3 @@ pub fn fonts(command: &FontsCommand) -> StrResult<()> { Ok(()) } - -/// Searches for fonts. -pub struct FontSearcher { - /// Metadata about all discovered fonts. - pub book: FontBook, - /// Slots that the fonts are loaded into. - pub fonts: Vec, -} - -/// Holds details about the location of a font and lazily the font itself. -pub struct FontSlot { - /// The path at which the font can be found on the system. - path: PathBuf, - /// The index of the font in its collection. Zero if the path does not point - /// to a collection. - index: u32, - /// The lazily loaded font. - font: OnceLock>, -} - -impl FontSlot { - /// Get the font for this slot. - pub fn get(&self) -> Option { - self.font - .get_or_init(|| { - let _scope = TimingScope::new("load font", None); - let data = fs::read(&self.path).ok()?.into(); - Font::new(data, self.index) - }) - .clone() - } -} - -impl FontSearcher { - /// Create a new, empty system searcher. - pub fn new() -> Self { - Self { book: FontBook::new(), fonts: vec![] } - } - - /// Search everything that is available. - pub fn search(&mut self, font_paths: &[PathBuf], ignore_system_fonts: bool) { - let mut db = Database::new(); - - // Font paths have highest priority. - for path in font_paths { - db.load_fonts_dir(path); - } - - if !ignore_system_fonts { - // System fonts have second priority. - db.load_system_fonts(); - } - - for face in db.faces() { - let path = match &face.source { - Source::File(path) | Source::SharedFile(path, _) => path, - // We never add binary sources to the database, so there - // shouln't be any. - Source::Binary(_) => continue, - }; - - let info = db - .with_face_data(face.id, FontInfo::new) - .expect("database must contain this font"); - - if let Some(info) = info { - self.book.push(info); - self.fonts.push(FontSlot { - path: path.clone(), - index: face.index, - font: OnceLock::new(), - }); - } - } - - // Embedded fonts have lowest priority. - #[cfg(feature = "embed-fonts")] - self.add_embedded(); - } - - /// Add fonts that are embedded in the binary. - #[cfg(feature = "embed-fonts")] - fn add_embedded(&mut self) { - for data in typst_assets::fonts() { - let buffer = typst::foundations::Bytes::from_static(data); - for (i, font) in Font::iter(buffer).enumerate() { - self.book.push(font.info().clone()); - self.fonts.push(FontSlot { - path: PathBuf::new(), - index: i as u32, - font: OnceLock::from(Some(font)), - }); - } - } - } -} diff --git a/crates/typst-cli/src/init.rs b/crates/typst-cli/src/init.rs index cb6b66275..842419fc1 100644 --- a/crates/typst-cli/src/init.rs +++ b/crates/typst-cli/src/init.rs @@ -10,11 +10,12 @@ use typst::syntax::package::{ }; use crate::args::InitCommand; -use crate::package::PackageStorage; +use crate::download::PrintDownload; +use crate::package; /// Execute an initialization command. pub fn init(command: &InitCommand) -> StrResult<()> { - let package_storage = PackageStorage::from_args(&command.package_storage_args); + let package_storage = package::storage(&command.package_storage_args); // Parse the package specification. If the user didn't specify the version, // we try to figure it out automatically by downloading the package index @@ -28,7 +29,8 @@ pub fn init(command: &InitCommand) -> StrResult<()> { })?; // Find or download the package. - let package_path = package_storage.prepare_package(&spec)?; + let package_path = + package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?; // Parse the manifest. let manifest = parse_manifest(&package_path)?; diff --git a/crates/typst-cli/src/package.rs b/crates/typst-cli/src/package.rs index bd5dd5493..b4965f89d 100644 --- a/crates/typst-cli/src/package.rs +++ b/crates/typst-cli/src/package.rs @@ -1,169 +1,13 @@ -use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use typst_kit::package::PackageStorage; use crate::args::PackageStorageArgs; -use codespan_reporting::term::{self, termcolor}; -use ecow::eco_format; -use once_cell::sync::OnceCell; -use termcolor::WriteColor; -use typst::diag::{bail, PackageError, PackageResult, StrResult}; -use typst::syntax::package::{ - PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec, -}; +use crate::download; -use crate::download::{download, download_with_progress}; -use crate::terminal; - -const HOST: &str = "https://packages.typst.org"; -const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; - -/// Holds information about where packages should be stored. -pub struct PackageStorage { - pub package_cache_path: Option, - pub package_path: Option, - index: OnceCell>, -} - -impl PackageStorage { - pub fn from_args(args: &PackageStorageArgs) -> Self { - let package_cache_path = args.package_cache_path.clone().or_else(|| { - dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR)) - }); - let package_path = args.package_path.clone().or_else(|| { - dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR)) - }); - Self { - package_cache_path, - package_path, - index: OnceCell::new(), - } - } - - /// Make a package available in the on-disk cache. - pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult { - let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); - - if let Some(packages_dir) = &self.package_path { - let dir = packages_dir.join(&subdir); - if dir.exists() { - return Ok(dir); - } - } - - if let Some(cache_dir) = &self.package_cache_path { - let dir = cache_dir.join(&subdir); - if dir.exists() { - return Ok(dir); - } - - // Download from network if it doesn't exist yet. - if spec.namespace == "preview" { - self.download_package(spec, &dir)?; - if dir.exists() { - return Ok(dir); - } - } - } - - Err(PackageError::NotFound(spec.clone())) - } - - /// Try to determine the latest version of a package. - pub fn determine_latest_version( - &self, - spec: &VersionlessPackageSpec, - ) -> StrResult { - if spec.namespace == "preview" { - // For `@preview`, download the package index and find the latest - // version. - self.download_index()? - .iter() - .filter(|package| package.name == spec.name) - .map(|package| package.version) - .max() - .ok_or_else(|| eco_format!("failed to find package {spec}")) - } else { - // For other namespaces, search locally. We only search in the data - // directory and not the cache directory, because the latter is not - // intended for storage of local packages. - let subdir = format!("{}/{}", spec.namespace, spec.name); - self.package_path - .iter() - .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) - .flatten() - .filter_map(|entry| entry.ok()) - .map(|entry| entry.path()) - .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok()) - .max() - .ok_or_else(|| eco_format!("please specify the desired version")) - } - } -} - -impl PackageStorage { - /// Download a package over the network. - fn download_package( - &self, - spec: &PackageSpec, - package_dir: &Path, - ) -> PackageResult<()> { - // The `@preview` namespace is the only namespace that supports on-demand - // fetching. - assert_eq!(spec.namespace, "preview"); - - let url = format!("{HOST}/preview/{}-{}.tar.gz", spec.name, spec.version); - - print_downloading(spec).unwrap(); - - let data = match download_with_progress(&url) { - Ok(data) => data, - Err(ureq::Error::Status(404, _)) => { - if let Ok(version) = self.determine_latest_version(&spec.versionless()) { - return Err(PackageError::VersionNotFound(spec.clone(), version)); - } else { - return Err(PackageError::NotFound(spec.clone())); - } - } - Err(err) => { - return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))) - } - }; - - let decompressed = flate2::read::GzDecoder::new(data.as_slice()); - tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| { - fs::remove_dir_all(package_dir).ok(); - PackageError::MalformedArchive(Some(eco_format!("{err}"))) - }) - } - - /// Download the `@preview` package index. - /// - /// To avoid downloading the index multiple times, the result is cached. - fn download_index(&self) -> StrResult<&Vec> { - self.index.get_or_try_init(|| { - let url = format!("{HOST}/preview/index.json"); - match download(&url) { - Ok(response) => response - .into_json() - .map_err(|err| eco_format!("failed to parse package index: {err}")), - Err(ureq::Error::Status(404, _)) => { - bail!("failed to fetch package index (not found)") - } - Err(err) => bail!("failed to fetch package index ({err})"), - } - }) - } -} - -/// Print that a package downloading is happening. -fn print_downloading(spec: &PackageSpec) -> io::Result<()> { - let styles = term::Styles::default(); - - let mut out = terminal::out(); - out.set_color(&styles.header_help)?; - write!(out, "downloading")?; - - out.reset()?; - writeln!(out, " {spec}") +/// Returns a new package storage for the given args. +pub fn storage(args: &PackageStorageArgs) -> PackageStorage { + PackageStorage::new( + args.package_cache_path.clone(), + args.package_path.clone(), + download::downloader(), + ) } diff --git a/crates/typst-cli/src/update.rs b/crates/typst-cli/src/update.rs index fa7c3a348..adec4a2ce 100644 --- a/crates/typst-cli/src/update.rs +++ b/crates/typst-cli/src/update.rs @@ -7,11 +7,12 @@ use semver::Version; use serde::Deserialize; use tempfile::NamedTempFile; use typst::diag::{bail, StrResult}; +use typst_kit::download::Downloader; use xz2::bufread::XzDecoder; use zip::ZipArchive; use crate::args::UpdateCommand; -use crate::download::{download, download_with_progress}; +use crate::download::{self, PrintDownload}; const TYPST_GITHUB_ORG: &str = "typst"; const TYPST_REPO: &str = "typst"; @@ -68,13 +69,15 @@ pub fn update(command: &UpdateCommand) -> StrResult<()> { fs::copy(current_exe, &backup_path) .map_err(|err| eco_format!("failed to create backup ({err})"))?; - let release = Release::from_tag(command.version.as_ref())?; + let downloader = download::downloader(); + + let release = Release::from_tag(command.version.as_ref(), &downloader)?; if !update_needed(&release)? && !command.force { eprintln!("Already up-to-date."); return Ok(()); } - let binary_data = release.download_binary(needed_asset()?)?; + let binary_data = release.download_binary(needed_asset()?, &downloader)?; let mut temp_exe = NamedTempFile::new() .map_err(|err| eco_format!("failed to create temporary file ({err})"))?; temp_exe @@ -106,7 +109,10 @@ struct Release { impl Release { /// Download the target release, or latest if version is `None`, from the /// Typst repository. - pub fn from_tag(tag: Option<&Version>) -> StrResult { + pub fn from_tag( + tag: Option<&Version>, + downloader: &Downloader, + ) -> StrResult { let url = match tag { Some(tag) => format!( "https://api.github.com/repos/{TYPST_GITHUB_ORG}/{TYPST_REPO}/releases/tags/v{tag}" @@ -116,7 +122,7 @@ impl Release { ), }; - match download(&url) { + match downloader.download(&url) { Ok(response) => response.into_json().map_err(|err| { eco_format!("failed to parse release information ({err})") }), @@ -130,15 +136,21 @@ impl Release { /// Download the binary from a given [`Release`] and select the /// corresponding asset for this target platform, returning the raw binary /// data. - pub fn download_binary(&self, asset_name: &str) -> StrResult> { + pub fn download_binary( + &self, + asset_name: &str, + downloader: &Downloader, + ) -> StrResult> { let asset = self .assets .iter() .find(|a| a.name.starts_with(asset_name)) .ok_or("could not find release for your target platform")?; - eprintln!("Downloading release ..."); - let data = match download_with_progress(&asset.browser_download_url) { + let data = match downloader.download_with_progress( + &asset.browser_download_url, + &mut PrintDownload("release"), + ) { Ok(data) => data, Err(ureq::Error::Status(404, _)) => { bail!("asset not found (searched for {})", asset.name); diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 5a0814a86..70c633550 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -14,12 +14,14 @@ use typst::syntax::{FileId, Source, VirtualPath}; use typst::text::{Font, FontBook}; use typst::utils::LazyHash; use typst::{Library, World}; +use typst_kit::fonts::{FontSlot, Fonts}; +use typst_kit::package::PackageStorage; use typst_timing::{timed, TimingScope}; use crate::args::{Input, SharedArgs}; use crate::compile::ExportCache; -use crate::fonts::{FontSearcher, FontSlot}; -use crate::package::PackageStorage; +use crate::download::PrintDownload; +use crate::package; /// Static `FileId` allocated for stdin. /// This is to ensure that a file is read in the correct way. @@ -110,26 +112,24 @@ impl SystemWorld { Library::builder().with_inputs(inputs).build() }; - let mut searcher = FontSearcher::new(); - searcher - .search(&command.font_args.font_paths, command.font_args.ignore_system_fonts); + let fonts = Fonts::searcher() + .include_system_fonts(command.font_args.ignore_system_fonts) + .search_with(&command.font_args.font_paths); let now = match command.creation_timestamp { Some(time) => Now::Fixed(time), None => Now::System(OnceLock::new()), }; - let package_storage = PackageStorage::from_args(&command.package_storage_args); - Ok(Self { workdir: std::env::current_dir().ok(), root, main, library: LazyHash::new(library), - book: LazyHash::new(searcher.book), - fonts: searcher.fonts, + book: LazyHash::new(fonts.book), + fonts: fonts.fonts, slots: Mutex::new(HashMap::new()), - package_storage, + package_storage: package::storage(&command.package_storage_args), now, export_cache: ExportCache::new(), }) @@ -378,7 +378,7 @@ fn system_path( let buf; let mut root = project_root; if let Some(spec) = id.package() { - buf = package_storage.prepare_package(spec)?; + buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?; root = &buf; } diff --git a/crates/typst-kit/Cargo.toml b/crates/typst-kit/Cargo.toml new file mode 100644 index 000000000..9c13ceed4 --- /dev/null +++ b/crates/typst-kit/Cargo.toml @@ -0,0 +1,57 @@ +[package] +name = "typst-kit" +description = "Common utilities for Typst tooling." +version = { workspace = true } +rust-version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +readme = { workspace = true } + +[dependencies] +typst = { workspace = true } +typst-assets = { workspace = true, optional = true } +typst-timing = { workspace = true } +typst-utils = { workspace = true } +ecow = { workspace = true } +env_proxy = { workspace = true, optional = true } +dirs = { workspace = true, optional = true } +flate2 = { workspace = true, optional = true } +fontdb = { workspace = true, optional = true } +native-tls = { workspace = true, optional = true } +once_cell = { workspace = true } +tar = { workspace = true, optional = true } +ureq = { workspace = true, optional = true } + +# Explicitly depend on OpenSSL if applicable, so that we can add the +# `openssl/vendored` feature to it if `vendor-openssl` is enabled. +[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies] +openssl = { workspace = true } + +[features] +default = ["fonts", "packages"] + +# Add font loading utilitites +fonts = ["dep:fontdb", "fontdb/memmap", "fontdb/fontconfig"] + +# Add generic downloading utilities +downloads = ["dep:env_proxy", "dep:native-tls", "dep:ureq"] + +# Add package downloading utilities, implies `downloads` +packages = ["downloads", "dep:dirs", "dep:flate2", "dep:tar"] + +# Embeds some fonts into the binary: +# - For text: Linux Libertine, New Computer Modern +# - For math: New Computer Modern Math +# - For code: Deja Vu Sans Mono +# +# Implies `fonts` +embed-fonts = ["fonts", "dep:typst-assets", "typst-assets/fonts"] + +# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds. +vendor-openssl = ["openssl/vendored"] + +[lints] +workspace = true diff --git a/crates/typst-kit/src/download.rs b/crates/typst-kit/src/download.rs new file mode 100644 index 000000000..9aff4dc02 --- /dev/null +++ b/crates/typst-kit/src/download.rs @@ -0,0 +1,259 @@ +// Acknowledgement: +// Closely modelled after rustup's `DownloadTracker`. +// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs + +//! Helpers for making various web requests with status reporting. These are +//! primarily used for communicating with package registries. + +use std::collections::VecDeque; +use std::fmt::Debug; +use std::io::{self, ErrorKind, Read}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use ecow::EcoString; +use native_tls::{Certificate, TlsConnector}; +use once_cell::sync::OnceCell; +use ureq::Response; + +/// Manages progress reporting for downloads. +pub trait Progress { + /// Invoked when a download is started. + fn print_start(&mut self); + + /// Invoked repeatedly while a download is ongoing. + fn print_progress(&mut self, state: &DownloadState); + + /// Invoked when a download is finished. + fn print_finish(&mut self, state: &DownloadState); +} + +/// An implementation of [`Progress`] whth no-op reporting, i.e. reporting +/// events are swallowed. +pub struct ProgressSink; + +impl Progress for ProgressSink { + fn print_start(&mut self) {} + fn print_progress(&mut self, _: &DownloadState) {} + fn print_finish(&mut self, _: &DownloadState) {} +} + +/// The current state of an in progress or finished download. +#[derive(Debug)] +pub struct DownloadState { + /// The expected amount of bytes to download, `None` if the response header + /// was not set. + pub content_len: Option, + /// The total amount of downloaded bytes until now. + pub total_downloaded: usize, + /// A backlog of the amount of downloaded bytes each second. + pub bytes_per_second: VecDeque, + /// The download starting instant. + pub start_time: Instant, +} + +/// A minimal https client for downloading various resources. +pub struct Downloader { + user_agent: EcoString, + cert_path: Option, + cert: OnceCell, +} + +impl Downloader { + /// Crates a new downloader with the given user agent and no certificate. + pub fn new(user_agent: impl Into) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: None, + cert: OnceCell::new(), + } + } + + /// Crates a new downloader with the given user agent and certificate path. + /// + /// If the certificate cannot be read it is set to `None`. + pub fn with_path(user_agent: impl Into, cert_path: PathBuf) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: Some(cert_path), + cert: OnceCell::new(), + } + } + + /// Crates a new downloader with the given user agent and certificate. + pub fn with_cert(user_agent: impl Into, cert: Certificate) -> Self { + Self { + user_agent: user_agent.into(), + cert_path: None, + cert: OnceCell::with_value(cert), + } + } + + /// Returns the certificate this client is using, if a custom certificate + /// is used it is loaded on first access. + /// + /// - Returns `None` if `--cert` and `TYPST_CERT` are not set. + /// - Returns `Some(Ok(cert))` if the certificate was loaded successfully. + /// - Returns `Some(Err(err))` if an error occurred while loading the certificate. + pub fn cert(&self) -> Option> { + self.cert_path.as_ref().map(|path| { + self.cert.get_or_try_init(|| { + let pem = std::fs::read(path)?; + Certificate::from_pem(&pem).map_err(io::Error::other) + }) + }) + } + + /// Download binary data from the given url. + #[allow(clippy::result_large_err)] + pub fn download(&self, url: &str) -> Result { + let mut builder = ureq::AgentBuilder::new(); + let mut tls = TlsConnector::builder(); + + // Set user agent. + builder = builder.user_agent(&self.user_agent); + + // Get the network proxy config from the environment and apply it. + if let Some(proxy) = env_proxy::for_url_str(url) + .to_url() + .and_then(|url| ureq::Proxy::new(url).ok()) + { + builder = builder.proxy(proxy); + } + + // Apply a custom CA certificate if present. + if let Some(cert) = self.cert() { + tls.add_root_certificate(cert?.clone()); + } + + // Configure native TLS. + let connector = + tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?; + builder = builder.tls_connector(Arc::new(connector)); + + builder.build().get(url).call() + } + + /// Download binary data from the given url and report its progress. + #[allow(clippy::result_large_err)] + pub fn download_with_progress( + &self, + url: &str, + progress: &mut dyn Progress, + ) -> Result, ureq::Error> { + progress.print_start(); + let response = self.download(url)?; + Ok(RemoteReader::from_response(response, progress).download()?) + } +} + +impl Debug for Downloader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Downloader") + .field("user_agent", &self.user_agent) + .field("cert_path", &self.cert_path) + .field( + "cert", + &self + .cert + .get() + .map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))), + ) + .finish() + } +} + +/// Keep track of this many download speed samples. +const SAMPLES: usize = 5; + +/// A wrapper around [`ureq::Response`] that reads the response body in chunks +/// over a websocket and reports its progress. +struct RemoteReader<'p> { + /// The reader returned by the ureq::Response. + reader: Box, + /// The download state, holding download metadata for progress reporting. + state: DownloadState, + /// The instant at which progress was last reported. + last_progress: Option, + /// A trait object used to report download progress. + progress: &'p mut dyn Progress, +} + +impl<'p> RemoteReader<'p> { + /// Wraps a [`ureq::Response`] and prepares it for downloading. + /// + /// The 'Content-Length' header is used as a size hint for read + /// optimization, if present. + fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self { + let content_len: Option = response + .header("Content-Length") + .and_then(|header| header.parse().ok()); + + Self { + reader: response.into_reader(), + last_progress: None, + state: DownloadState { + content_len, + total_downloaded: 0, + bytes_per_second: VecDeque::with_capacity(SAMPLES), + start_time: Instant::now(), + }, + progress, + } + } + + /// Download the body's content as raw bytes while reporting download + /// progress. + fn download(mut self) -> io::Result> { + let mut buffer = vec![0; 8192]; + let mut data = match self.state.content_len { + Some(content_len) => Vec::with_capacity(content_len), + None => Vec::with_capacity(8192), + }; + + let mut downloaded_this_sec = 0; + loop { + let read = match self.reader.read(&mut buffer) { + Ok(0) => break, + Ok(n) => n, + // If the data is not yet ready but will be available eventually + // keep trying until we either get an actual error, receive data + // or an Ok(0). + Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + + data.extend(&buffer[..read]); + + let last_printed = match self.last_progress { + Some(prev) => prev, + None => { + let current_time = Instant::now(); + self.last_progress = Some(current_time); + current_time + } + }; + let elapsed = Instant::now().saturating_duration_since(last_printed); + + downloaded_this_sec += read; + self.state.total_downloaded += read; + + if elapsed >= Duration::from_secs(1) { + if self.state.bytes_per_second.len() == SAMPLES { + self.state.bytes_per_second.pop_back(); + } + + self.state.bytes_per_second.push_front(downloaded_this_sec); + downloaded_this_sec = 0; + + self.progress.print_progress(&self.state); + self.last_progress = Some(Instant::now()); + } + } + + self.progress.print_finish(&self.state); + + Ok(data) + } +} diff --git a/crates/typst-kit/src/fonts.rs b/crates/typst-kit/src/fonts.rs new file mode 100644 index 000000000..07a41f4b2 --- /dev/null +++ b/crates/typst-kit/src/fonts.rs @@ -0,0 +1,216 @@ +//! Default implementation for searching local and system installed fonts as +//! well as loading embedded default fonts. +//! +//! # Embedded fonts +//! The following fonts are available as embedded fonts via the `embed-fonts` +//! feature flag: +//! - For text: Linux Libertine, New Computer Modern +//! - For math: New Computer Modern Math +//! - For code: Deja Vu Sans Mono + +use std::path::PathBuf; +use std::sync::OnceLock; +use std::{fs, path::Path}; + +use fontdb::{Database, Source}; +use typst::text::{Font, FontBook, FontInfo}; +use typst_timing::TimingScope; + +/// Holds details about the location of a font and lazily the font itself. +#[derive(Debug)] +pub struct FontSlot { + /// The path at which the font can be found on the system. + path: Option, + /// The index of the font in its collection. Zero if the path does not point + /// to a collection. + index: u32, + /// The lazily loaded font. + font: OnceLock>, +} + +impl FontSlot { + /// Returns the path at which the font can be found on the system, or `None` + /// if the font was embedded. + pub fn path(&self) -> Option<&Path> { + self.path.as_deref() + } + + /// Returns the index of the font in its collection. Zero if the path does + /// not point to a collection. + pub fn index(&self) -> u32 { + self.index + } + + /// Get the font for this slot. This loads the font into memory on first + /// access. + pub fn get(&self) -> Option { + self.font + .get_or_init(|| { + let _scope = TimingScope::new("load font", None); + let data = fs::read( + self.path + .as_ref() + .expect("`path` is not `None` if `font` is uninitialized"), + ) + .ok()? + .into(); + Font::new(data, self.index) + }) + .clone() + } +} + +/// The result of a font search, created by calling [`FontSearcher::search`]. +#[derive(Debug)] +pub struct Fonts { + /// Metadata about all discovered fonts. + pub book: FontBook, + /// Slots that the fonts are loaded into. + pub fonts: Vec, +} + +impl Fonts { + /// Creates a new font searcer with the default settings. + pub fn searcher() -> FontSearcher { + FontSearcher::new() + } +} + +/// Searches for fonts. +/// +/// Fonts are added in the following order (descending priority): +/// 1. Font directories +/// 2. System fonts (if included & enabled) +/// 3. Embedded fonts (if enabled) +#[derive(Debug)] +pub struct FontSearcher { + db: Database, + include_system_fonts: bool, + #[cfg(feature = "embed-fonts")] + include_embedded_fonts: bool, + book: FontBook, + fonts: Vec, +} + +impl FontSearcher { + /// Create a new, empty system searcher. The searcher is created with the + /// default configuration, it will include embedded fonts and system fonts. + pub fn new() -> Self { + Self { + db: Database::new(), + include_system_fonts: true, + #[cfg(feature = "embed-fonts")] + include_embedded_fonts: true, + book: FontBook::new(), + fonts: vec![], + } + } + + /// Whether to search for and load system fonts, defaults to `true`. + pub fn include_system_fonts(&mut self, value: bool) -> &mut Self { + self.include_system_fonts = value; + self + } + + /// Whether to load embedded fonts, defaults to `true`. + #[cfg(feature = "embed-fonts")] + pub fn include_embedded_fonts(&mut self, value: bool) -> &mut Self { + self.include_embedded_fonts = value; + self + } + + /// Start searching for and loading fonts. To additionally load fonts + /// from specific directories, use [`search_with`][Self::search_with]. + /// + /// # Examples + /// ```no_run + /// # use typst_kit::fonts::FontSearcher; + /// let fonts = FontSearcher::new() + /// .include_system_fonts(true) + /// .search(); + /// ``` + pub fn search(&mut self) -> Fonts { + self.search_with::<_, &str>([]) + } + + /// Start searching for and loading fonts, with additional directories. + /// + /// # Examples + /// ```no_run + /// # use typst_kit::fonts::FontSearcher; + /// let fonts = FontSearcher::new() + /// .include_system_fonts(true) + /// .search_with(["./assets/fonts/"]); + /// ``` + pub fn search_with(&mut self, font_dirs: I) -> Fonts + where + I: IntoIterator, + P: AsRef, + { + // Font paths have highest priority. + for path in font_dirs { + self.db.load_fonts_dir(path); + } + + if self.include_system_fonts { + // System fonts have second priority. + self.db.load_system_fonts(); + } + + for face in self.db.faces() { + let path = match &face.source { + Source::File(path) | Source::SharedFile(path, _) => path, + // We never add binary sources to the database, so there + // shouln't be any. + Source::Binary(_) => continue, + }; + + let info = self + .db + .with_face_data(face.id, FontInfo::new) + .expect("database must contain this font"); + + if let Some(info) = info { + self.book.push(info); + self.fonts.push(FontSlot { + path: Some(path.clone()), + index: face.index, + font: OnceLock::new(), + }); + } + } + + // Embedded fonts have lowest priority. + #[cfg(feature = "embed-fonts")] + if self.include_embedded_fonts { + self.add_embedded(); + } + + Fonts { + book: std::mem::take(&mut self.book), + fonts: std::mem::take(&mut self.fonts), + } + } + + /// Add fonts that are embedded in the binary. + #[cfg(feature = "embed-fonts")] + fn add_embedded(&mut self) { + for data in typst_assets::fonts() { + let buffer = typst::foundations::Bytes::from_static(data); + for (i, font) in Font::iter(buffer).enumerate() { + self.book.push(font.info().clone()); + self.fonts.push(FontSlot { + path: None, + index: i as u32, + font: OnceLock::from(Some(font)), + }); + } + } + } +} + +impl Default for FontSearcher { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/typst-kit/src/lib.rs b/crates/typst-kit/src/lib.rs new file mode 100644 index 000000000..4301727fb --- /dev/null +++ b/crates/typst-kit/src/lib.rs @@ -0,0 +1,27 @@ +//! Typst-kit contains various default implementations of functionality used in +//! typst-cli. It is intended as a single source of truth for things like font +//! searching, package downloads and more. Each component of typst-kit is +//! optional, but enabled by default. +//! +//! # Components +//! - [fonts] contains a default implementation for searching local and system +//! installed fonts. It is enabled by the `fonts` feature flag, additionally +//! the `embed-fonts` feature can be used to embed the Typst default fonts. +//! - For text: Linux Libertine, New Computer Modern +//! - For math: New Computer Modern Math +//! - For code: Deja Vu Sans Mono +//! - [download] contains functionality for making simple web requests with +//! status reporting, useful for downloading packages from package registires. +//! It is enabled by the `downloads` feature flag, additionally the +//! `vendor-openssl` can be used on operating systems other than macOS and +//! Windows to vendor OpenSSL when building. +//! - [package] contains package storage and downloading functionality based on +//! [download]. It is enabled by the `packages` feature flag and implies the +//! `downloads` feature flag. + +#[cfg(feature = "downloads")] +pub mod download; +#[cfg(feature = "fonts")] +pub mod fonts; +#[cfg(feature = "packages")] +pub mod package; diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs new file mode 100644 index 000000000..ad69df01d --- /dev/null +++ b/crates/typst-kit/src/package.rs @@ -0,0 +1,179 @@ +//! Download and unpack packages and package indices. + +use std::fs; +use std::path::{Path, PathBuf}; + +use ecow::eco_format; +use once_cell::sync::OnceCell; +use typst::diag::{bail, PackageError, PackageResult, StrResult}; +use typst::syntax::package::{ + PackageInfo, PackageSpec, PackageVersion, VersionlessPackageSpec, +}; + +use crate::download::{Downloader, Progress}; + +/// The default Typst registry. +pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org"; + +/// The default packages sub directory within the package and package cache paths. +pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; + +/// Holds information about where packages should be stored and downloads them +/// on demand, if possible. +#[derive(Debug)] +pub struct PackageStorage { + /// The path at which non-local packages should be stored when downloaded. + package_cache_path: Option, + /// The path at which local packages are stored. + package_path: Option, + /// The downloader used for fetching the index and packages. + downloader: Downloader, + /// The cached index of the preview namespace. + index: OnceCell>, +} + +impl PackageStorage { + /// Creates a new package storage for the given package paths. Falls back to + /// the recommended XDG directories if they are `None`. + pub fn new( + package_cache_path: Option, + package_path: Option, + downloader: Downloader, + ) -> Self { + Self { + package_cache_path: package_cache_path.or_else(|| { + dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR)) + }), + package_path: package_path.or_else(|| { + dirs::data_dir().map(|data_dir| data_dir.join(DEFAULT_PACKAGES_SUBDIR)) + }), + downloader, + index: OnceCell::new(), + } + } + + /// Returns a the path at which non-local packages should be stored when + /// downloaded. + pub fn package_cache_path(&self) -> Option<&Path> { + self.package_cache_path.as_deref() + } + + /// Returns a the path at which local packages are stored. + pub fn package_path(&self) -> Option<&Path> { + self.package_path.as_deref() + } + + /// Make a package available in the on-disk. + pub fn prepare_package( + &self, + spec: &PackageSpec, + progress: &mut dyn Progress, + ) -> PackageResult { + let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); + + if let Some(packages_dir) = &self.package_path { + let dir = packages_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + } + + if let Some(cache_dir) = &self.package_cache_path { + let dir = cache_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + + // Download from network if it doesn't exist yet. + self.download_package(spec, &dir, progress)?; + if dir.exists() { + return Ok(dir); + } + } + + Err(PackageError::NotFound(spec.clone())) + } + + /// Try to determine the latest version of a package. + pub fn determine_latest_version( + &self, + spec: &VersionlessPackageSpec, + ) -> StrResult { + if spec.namespace == "preview" { + // For `@preview`, download the package index and find the latest + // version. + self.download_index()? + .iter() + .filter(|package| package.name == spec.name) + .map(|package| package.version) + .max() + .ok_or_else(|| eco_format!("failed to find package {spec}")) + } else { + // For other namespaces, search locally. We only search in the data + // directory and not the cache directory, because the latter is not + // intended for storage of local packages. + let subdir = format!("{}/{}", spec.namespace, spec.name); + self.package_path + .iter() + .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok()) + .flatten() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.path()) + .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok()) + .max() + .ok_or_else(|| eco_format!("please specify the desired version")) + } + } + + /// Download the package index. The result of this is cached for efficiency. + pub fn download_index(&self) -> StrResult<&Vec> { + self.index.get_or_try_init(|| { + let url = format!("{DEFAULT_REGISTRY}/preview/index.json"); + match self.downloader.download(&url) { + Ok(response) => response + .into_json() + .map_err(|err| eco_format!("failed to parse package index: {err}")), + Err(ureq::Error::Status(404, _)) => { + bail!("failed to fetch package index (not found)") + } + Err(err) => bail!("failed to fetch package index ({err})"), + } + }) + } + + /// Download a package over the network. + /// + /// # Panics + /// Panics if the package spec namespace isn't `preview`. + pub fn download_package( + &self, + spec: &PackageSpec, + package_dir: &Path, + progress: &mut dyn Progress, + ) -> PackageResult<()> { + assert_eq!(spec.namespace, "preview"); + + let url = + format!("{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz", spec.name, spec.version); + + let data = match self.downloader.download_with_progress(&url, progress) { + Ok(data) => data, + Err(ureq::Error::Status(404, _)) => { + if let Ok(version) = self.determine_latest_version(&spec.versionless()) { + return Err(PackageError::VersionNotFound(spec.clone(), version)); + } else { + return Err(PackageError::NotFound(spec.clone())); + } + } + Err(err) => { + return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))) + } + }; + + let decompressed = flate2::read::GzDecoder::new(data.as_slice()); + tar::Archive::new(decompressed).unpack(package_dir).map_err(|err| { + fs::remove_dir_all(package_dir).ok(); + PackageError::MalformedArchive(Some(eco_format!("{err}"))) + }) + } +}