From ddec8feab31886cae2133bd3929b8000bcaa66e4 Mon Sep 17 00:00:00 2001 From: stelzo Date: Tue, 21 Jan 2025 23:13:33 +0100 Subject: [PATCH 1/5] add vendoring --- crates/typst-cli/src/args.rs | 19 ++++++ crates/typst-cli/src/init.rs | 4 +- crates/typst-cli/src/main.rs | 2 + crates/typst-cli/src/vendor.rs | 100 ++++++++++++++++++++++++++++++++ crates/typst-cli/src/world.rs | 6 +- crates/typst-kit/src/package.rs | 12 ++++ 6 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 crates/typst-cli/src/vendor.rs diff --git a/crates/typst-cli/src/args.rs b/crates/typst-cli/src/args.rs index d6855d100..0b09094ff 100644 --- a/crates/typst-cli/src/args.rs +++ b/crates/typst-cli/src/args.rs @@ -75,6 +75,9 @@ pub enum Command { /// Processes an input file to extract provided metadata. Query(QueryCommand), + /// Create a vendor directory with all used packages. + Vendor(VendorCommand), + /// Lists all discovered fonts in system and custom font paths. Fonts(FontsCommand), @@ -160,6 +163,22 @@ pub struct QueryCommand { pub process: ProcessArgs, } +/// Create a vendor directory with all used packages in the current directory. +#[derive(Debug, Clone, Parser)] +pub struct VendorCommand { + /// Path to input Typst file. Use `-` to read input from stdin. + #[clap(value_parser = input_value_parser(), value_hint = ValueHint::FilePath)] + pub input: Input, + + /// World arguments. + #[clap(flatten)] + pub world: WorldArgs, + + /// Processing arguments. + #[clap(flatten)] + pub process: ProcessArgs, +} + /// Lists all discovered fonts in system and custom font paths. #[derive(Debug, Clone, Parser)] pub struct FontsCommand { diff --git a/crates/typst-cli/src/init.rs b/crates/typst-cli/src/init.rs index 9a77fb470..0e0a5c646 100644 --- a/crates/typst-cli/src/init.rs +++ b/crates/typst-cli/src/init.rs @@ -28,9 +28,9 @@ pub fn init(command: &InitCommand) -> StrResult<()> { StrResult::Ok(spec.at(version)) })?; - // Find or download the package. + // Find or download the package. Vendoring does not make sense for initialization, so project_root is not needed. let package_path = - package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?; + package_storage.prepare_package(&spec, &mut PrintDownload(&spec), None)?; // Parse the manifest. let manifest = parse_manifest(&package_path)?; diff --git a/crates/typst-cli/src/main.rs b/crates/typst-cli/src/main.rs index 14f8a665d..4e313552e 100644 --- a/crates/typst-cli/src/main.rs +++ b/crates/typst-cli/src/main.rs @@ -12,6 +12,7 @@ mod terminal; mod timings; #[cfg(feature = "self-update")] mod update; +mod vendor; mod watch; mod world; @@ -69,6 +70,7 @@ fn dispatch() -> HintedStrResult<()> { Command::Watch(command) => crate::watch::watch(&mut timer, command)?, Command::Init(command) => crate::init::init(command)?, Command::Query(command) => crate::query::query(command)?, + Command::Vendor(command) => crate::vendor::vendor(command)?, Command::Fonts(command) => crate::fonts::fonts(command), Command::Update(command) => crate::update::update(command)?, } diff --git a/crates/typst-cli/src/vendor.rs b/crates/typst-cli/src/vendor.rs new file mode 100644 index 000000000..08bd4c1bd --- /dev/null +++ b/crates/typst-cli/src/vendor.rs @@ -0,0 +1,100 @@ +use std::fs::{create_dir, create_dir_all}; + +use ecow::eco_format; +use typst::{ + diag::{bail, HintedStrResult, Warned}, + layout::PagedDocument, +}; +use typst_kit::package::DEFAULT_PACKAGES_SUBDIR; + +use crate::{ + args::VendorCommand, compile::print_diagnostics, set_failed, world::SystemWorld, +}; +use typst::World; + +/// Execute a vendor command. +pub fn vendor(command: &VendorCommand) -> HintedStrResult<()> { + let mut world = SystemWorld::new(&command.input, &command.world, &command.process)?; + + // Reset everything and ensure that the main file is present. + world.reset(); + world.source(world.main()).map_err(|err| err.to_string())?; + + let Warned { output, warnings } = typst::compile::(&world); + + match output { + Ok(_) => { + copy_deps(&mut world)?; + print_diagnostics(&world, &[], &warnings, command.process.diagnostic_format) + .map_err(|err| eco_format!("failed to print diagnostics ({err})"))?; + } + + // Print diagnostics. + Err(errors) => { + set_failed(); + print_diagnostics( + &world, + &errors, + &warnings, + command.process.diagnostic_format, + ) + .map_err(|err| eco_format!("failed to print diagnostics ({err})"))?; + } + } + + Ok(()) +} + +fn copy_deps(world: &mut SystemWorld) -> HintedStrResult<()> { + let vendor_dir = world.workdir().join("vendor"); + + match vendor_dir.try_exists() { + Ok(false) => { + if let Err(err) = create_dir(vendor_dir.clone()) { + bail!("failed to create vendor directory: {:?}", err); + } + } + Err(err) => { + bail!("failed to check existence of vendor directory: {:?}", err); + } + _ => {} + } + + // Must iterate two times in total. As soon as the parent directory is created, + // world tries to read the subsequent files from the same package + // from the vendor directory since it is higher priority. + let all_deps = world + .dependencies() + .filter_map(|dep_path| { + let path = dep_path.to_str().unwrap(); + path.find(DEFAULT_PACKAGES_SUBDIR).map(|pos| { + let dependency_path = &path[pos + DEFAULT_PACKAGES_SUBDIR.len() + 1..]; + (dep_path.clone(), vendor_dir.join(dependency_path)) + }) + }) + .collect::>(); + + for (from_data_path, to_vendor_path) in all_deps { + if let Some(parent) = to_vendor_path.parent() { + match parent.try_exists() { + Ok(false) => { + if let Err(err) = create_dir_all(parent) { + bail!( + "failed to create package inside the vendor directory: {:?}", + err + ); + } + } + Err(err) => { + bail!("failed to check existence of a package inside the vendor directory: {:?}", err); + } + _ => {} + } + } + + if let Err(err) = std::fs::copy(from_data_path, to_vendor_path) { + bail!("failed to copy dependency to vendor directory: {:?}", err); + } + } + Ok(()) +} diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 12e80d273..c9058b5e9 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -381,7 +381,11 @@ fn system_path( let buf; let mut root = project_root; if let Some(spec) = id.package() { - buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?; + buf = package_storage.prepare_package( + spec, + &mut PrintDownload(&spec), + Some(root), + )?; root = &buf; } diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index e7eb71ee4..0ab8e943d 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -71,6 +71,7 @@ impl PackageStorage { &self, spec: &PackageSpec, progress: &mut dyn Progress, + project_root: Option<&Path>, ) -> PackageResult { let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); @@ -81,6 +82,17 @@ impl PackageStorage { } } + // Read from vendor dir if it exists. + if let Some(project_root) = project_root { + let vendor_dir = project_root.join("vendor"); + if let Ok(true) = vendor_dir.try_exists() { + let dir = vendor_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + } + } + if let Some(cache_dir) = &self.package_cache_path { let dir = cache_dir.join(&subdir); if dir.exists() { From 9256871d6234e191752da706a901b75f059e1051 Mon Sep 17 00:00:00 2001 From: stelzo Date: Tue, 21 Jan 2025 23:49:47 +0100 Subject: [PATCH 2/5] const vendor dir name --- crates/typst-cli/src/vendor.rs | 4 ++-- crates/typst-kit/src/package.rs | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/typst-cli/src/vendor.rs b/crates/typst-cli/src/vendor.rs index 08bd4c1bd..a6e36a55a 100644 --- a/crates/typst-cli/src/vendor.rs +++ b/crates/typst-cli/src/vendor.rs @@ -5,7 +5,7 @@ use typst::{ diag::{bail, HintedStrResult, Warned}, layout::PagedDocument, }; -use typst_kit::package::DEFAULT_PACKAGES_SUBDIR; +use typst_kit::package::{DEFAULT_PACKAGES_SUBDIR, DEFAULT_VENDOR_SUBDIR}; use crate::{ args::VendorCommand, compile::print_diagnostics, set_failed, world::SystemWorld, @@ -46,7 +46,7 @@ pub fn vendor(command: &VendorCommand) -> HintedStrResult<()> { } fn copy_deps(world: &mut SystemWorld) -> HintedStrResult<()> { - let vendor_dir = world.workdir().join("vendor"); + let vendor_dir = world.workdir().join(DEFAULT_VENDOR_SUBDIR); match vendor_dir.try_exists() { Ok(false) => { diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index 0ab8e943d..ea561f6a4 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -21,6 +21,9 @@ pub const DEFAULT_NAMESPACE: &str = "preview"; /// The default packages sub directory within the package and package cache paths. pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages"; +/// The default vendor sub directory within the project root. +pub const DEFAULT_VENDOR_SUBDIR: &str = "vendor"; + /// Holds information about where packages should be stored and downloads them /// on demand, if possible. #[derive(Debug)] @@ -84,7 +87,7 @@ impl PackageStorage { // Read from vendor dir if it exists. if let Some(project_root) = project_root { - let vendor_dir = project_root.join("vendor"); + let vendor_dir = project_root.join(DEFAULT_VENDOR_SUBDIR); if let Ok(true) = vendor_dir.try_exists() { let dir = vendor_dir.join(&subdir); if dir.exists() { From 476b79ddfc3e1744eb79a97776312fa373cb6e15 Mon Sep 17 00:00:00 2001 From: stelzo Date: Wed, 22 Jan 2025 01:19:19 +0100 Subject: [PATCH 3/5] add vendor dir name param --- crates/typst-cli/src/args.rs | 8 +++++++ crates/typst-cli/src/init.rs | 6 ++--- crates/typst-cli/src/package.rs | 6 ++++- crates/typst-cli/src/vendor.rs | 39 ++++++++++++++++++++------------- crates/typst-cli/src/world.rs | 18 +++++++-------- crates/typst-kit/src/package.rs | 10 ++++++--- 6 files changed, 56 insertions(+), 31 deletions(-) diff --git a/crates/typst-cli/src/args.rs b/crates/typst-cli/src/args.rs index 0b09094ff..32387985a 100644 --- a/crates/typst-cli/src/args.rs +++ b/crates/typst-cli/src/args.rs @@ -361,6 +361,14 @@ pub struct PackageArgs { value_name = "DIR" )] pub package_cache_path: Option, + + /// Custom vendor directory name. + #[clap( + long = "package-vendor-path", + env = "TYPST_PACKAGE_VENDOR_PATH", + value_name = "DIR" + )] + pub vendor_path: Option, } /// Common arguments to customize available fonts. diff --git a/crates/typst-cli/src/init.rs b/crates/typst-cli/src/init.rs index 0e0a5c646..c9da3434a 100644 --- a/crates/typst-cli/src/init.rs +++ b/crates/typst-cli/src/init.rs @@ -15,7 +15,7 @@ use crate::package; /// Execute an initialization command. pub fn init(command: &InitCommand) -> StrResult<()> { - let package_storage = package::storage(&command.package); + let package_storage = package::storage(&command.package, None); // Parse the package specification. If the user didn't specify the version, // we try to figure it out automatically by downloading the package index @@ -28,9 +28,9 @@ pub fn init(command: &InitCommand) -> StrResult<()> { StrResult::Ok(spec.at(version)) })?; - // Find or download the package. Vendoring does not make sense for initialization, so project_root is not needed. + // Find or download the package. Vendoring does not make sense for initialization, so vendor_dir is not needed. let package_path = - package_storage.prepare_package(&spec, &mut PrintDownload(&spec), None)?; + package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?; // Parse the manifest. let manifest = parse_manifest(&package_path)?; diff --git a/crates/typst-cli/src/package.rs b/crates/typst-cli/src/package.rs index 6099ecaa9..48e95e97f 100644 --- a/crates/typst-cli/src/package.rs +++ b/crates/typst-cli/src/package.rs @@ -1,13 +1,17 @@ +use std::path::PathBuf; + use typst_kit::package::PackageStorage; use crate::args::PackageArgs; use crate::download; /// Returns a new package storage for the given args. -pub fn storage(args: &PackageArgs) -> PackageStorage { +pub fn storage(args: &PackageArgs, workdir: Option) -> PackageStorage { PackageStorage::new( + args.vendor_path.clone(), args.package_cache_path.clone(), args.package_path.clone(), download::downloader(), + workdir, ) } diff --git a/crates/typst-cli/src/vendor.rs b/crates/typst-cli/src/vendor.rs index a6e36a55a..41300e6e1 100644 --- a/crates/typst-cli/src/vendor.rs +++ b/crates/typst-cli/src/vendor.rs @@ -1,4 +1,7 @@ -use std::fs::{create_dir, create_dir_all}; +use std::{ + fs::{create_dir, create_dir_all}, + path::PathBuf, +}; use ecow::eco_format; use typst::{ @@ -24,7 +27,7 @@ pub fn vendor(command: &VendorCommand) -> HintedStrResult<()> { match output { Ok(_) => { - copy_deps(&mut world)?; + copy_deps(&mut world, &command.world.package.vendor_path)?; print_diagnostics(&world, &[], &warnings, command.process.diagnostic_format) .map_err(|err| eco_format!("failed to print diagnostics ({err})"))?; } @@ -45,20 +48,26 @@ pub fn vendor(command: &VendorCommand) -> HintedStrResult<()> { Ok(()) } -fn copy_deps(world: &mut SystemWorld) -> HintedStrResult<()> { - let vendor_dir = world.workdir().join(DEFAULT_VENDOR_SUBDIR); - - match vendor_dir.try_exists() { - Ok(false) => { - if let Err(err) = create_dir(vendor_dir.clone()) { - bail!("failed to create vendor directory: {:?}", err); +fn copy_deps( + world: &mut SystemWorld, + vendor_path: &Option, +) -> HintedStrResult<()> { + let vendor_dir = match vendor_path { + Some(path) => match path.canonicalize() { + Ok(path) => path, + Err(err) => { + if err.kind() == std::io::ErrorKind::NotFound { + if let Err(err) = create_dir(path) { + bail!("failed to create vendor directory: {:?}", err); + } + path.clone() + } else { + bail!("failed to canonicalize vendor directory path: {:?}", err); + } } - } - Err(err) => { - bail!("failed to check existence of vendor directory: {:?}", err); - } - _ => {} - } + }, + None => world.workdir().join(DEFAULT_VENDOR_SUBDIR), + }; // Must iterate two times in total. As soon as the parent directory is created, // world tries to read the subsequent files from the same package diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index c9058b5e9..60c182a7c 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -29,7 +29,7 @@ static STDIN_ID: LazyLock = /// A world that provides access to the operating system. pub struct SystemWorld { /// The working directory. - workdir: Option, + workdir: PathBuf, /// The root relative to which absolute paths are resolved. root: PathBuf, /// The input path. @@ -132,15 +132,18 @@ impl SystemWorld { None => Now::System(OnceLock::new()), }; + let env_workdir = std::env::current_dir().ok(); + let workdir = env_workdir.unwrap_or(PathBuf::from(".")); + Ok(Self { - workdir: std::env::current_dir().ok(), + workdir: workdir.clone(), root, main, library: LazyHash::new(library), book: LazyHash::new(fonts.book), fonts: fonts.fonts, slots: Mutex::new(HashMap::new()), - package_storage: package::storage(&world_args.package), + package_storage: package::storage(&world_args.package, Some(workdir)), now, }) } @@ -157,7 +160,7 @@ impl SystemWorld { /// The current working directory. pub fn workdir(&self) -> &Path { - self.workdir.as_deref().unwrap_or(Path::new(".")) + self.workdir.as_path() } /// Return all paths the last compilation depended on. @@ -380,12 +383,9 @@ fn system_path( // will be resolved. let buf; let mut root = project_root; + if let Some(spec) = id.package() { - buf = package_storage.prepare_package( - spec, - &mut PrintDownload(&spec), - Some(root), - )?; + buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?; root = &buf; } diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index ea561f6a4..17df96ea9 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -28,6 +28,8 @@ pub const DEFAULT_VENDOR_SUBDIR: &str = "vendor"; /// on demand, if possible. #[derive(Debug)] pub struct PackageStorage { + /// The path at which packages are stored by the vendor command. + package_vendor_path: Option, /// The path at which non-local packages should be stored when downloaded. package_cache_path: Option, /// The path at which local packages are stored. @@ -42,11 +44,15 @@ impl PackageStorage { /// Creates a new package storage for the given package paths. Falls back to /// the recommended XDG directories if they are `None`. pub fn new( + package_vendor_path: Option, package_cache_path: Option, package_path: Option, downloader: Downloader, + workdir: Option, ) -> Self { Self { + package_vendor_path: package_vendor_path + .or_else(|| workdir.map(|workdir| workdir.join(DEFAULT_VENDOR_SUBDIR))), package_cache_path: package_cache_path.or_else(|| { dirs::cache_dir().map(|cache_dir| cache_dir.join(DEFAULT_PACKAGES_SUBDIR)) }), @@ -74,7 +80,6 @@ impl PackageStorage { &self, spec: &PackageSpec, progress: &mut dyn Progress, - project_root: Option<&Path>, ) -> PackageResult { let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); @@ -86,8 +91,7 @@ impl PackageStorage { } // Read from vendor dir if it exists. - if let Some(project_root) = project_root { - let vendor_dir = project_root.join(DEFAULT_VENDOR_SUBDIR); + if let Some(vendor_dir) = &self.package_vendor_path { if let Ok(true) = vendor_dir.try_exists() { let dir = vendor_dir.join(&subdir); if dir.exists() { From a87b426e66a941d959d9b27733988702192e2fd0 Mon Sep 17 00:00:00 2001 From: stelzo Date: Wed, 22 Jan 2025 01:24:24 +0100 Subject: [PATCH 4/5] comment irrelevant now --- crates/typst-cli/src/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/typst-cli/src/init.rs b/crates/typst-cli/src/init.rs index c9da3434a..f7bdb9d1a 100644 --- a/crates/typst-cli/src/init.rs +++ b/crates/typst-cli/src/init.rs @@ -28,7 +28,7 @@ pub fn init(command: &InitCommand) -> StrResult<()> { StrResult::Ok(spec.at(version)) })?; - // Find or download the package. Vendoring does not make sense for initialization, so vendor_dir is not needed. + // Find or download the package. let package_path = package_storage.prepare_package(&spec, &mut PrintDownload(&spec))?; From c8e838036b1347066f5e4593ad47d586ed0d582f Mon Sep 17 00:00:00 2001 From: stelzo Date: Wed, 22 Jan 2025 18:05:54 +0100 Subject: [PATCH 5/5] vendor highest prio when searching --- crates/typst-cli/src/world.rs | 1 - crates/typst-kit/src/package.rs | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/crates/typst-cli/src/world.rs b/crates/typst-cli/src/world.rs index 60c182a7c..e4f2354c8 100644 --- a/crates/typst-cli/src/world.rs +++ b/crates/typst-cli/src/world.rs @@ -383,7 +383,6 @@ fn system_path( // will be resolved. let buf; let mut root = project_root; - if let Some(spec) = id.package() { buf = package_storage.prepare_package(spec, &mut PrintDownload(&spec))?; root = &buf; diff --git a/crates/typst-kit/src/package.rs b/crates/typst-kit/src/package.rs index 17df96ea9..a2e29ef29 100644 --- a/crates/typst-kit/src/package.rs +++ b/crates/typst-kit/src/package.rs @@ -83,13 +83,6 @@ impl PackageStorage { ) -> PackageResult { let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version); - if let Some(packages_dir) = &self.package_path { - let dir = packages_dir.join(&subdir); - if dir.exists() { - return Ok(dir); - } - } - // Read from vendor dir if it exists. if let Some(vendor_dir) = &self.package_vendor_path { if let Ok(true) = vendor_dir.try_exists() { @@ -100,6 +93,13 @@ impl PackageStorage { } } + if let Some(packages_dir) = &self.package_path { + let dir = packages_dir.join(&subdir); + if dir.exists() { + return Ok(dir); + } + } + if let Some(cache_dir) = &self.package_cache_path { let dir = cache_dir.join(&subdir); if dir.exists() {