Module loading system

Detects cyclic imports and loads each module only once per compilation.
This commit is contained in:
Laurenz 2021-05-29 15:45:57 +02:00
parent 9f77f09aac
commit e023bf2ac9
14 changed files with 276 additions and 112 deletions

View File

@ -24,8 +24,8 @@ fn benchmarks(c: &mut Criterion) {
let state = typst::exec::State::default();
for case in CASES {
let case = Path::new(case);
let name = case.file_stem().unwrap().to_string_lossy();
let path = Path::new(TYP_DIR).join(case);
let name = path.file_stem().unwrap().to_string_lossy();
macro_rules! bench {
($step:literal: $code:expr) => {
@ -39,18 +39,18 @@ fn benchmarks(c: &mut Criterion) {
}
// Prepare intermediate results, run warm and fill caches.
let src = std::fs::read_to_string(Path::new(TYP_DIR).join(case)).unwrap();
let parsed = Rc::new(parse(&src).output);
let evaluated = eval(&mut loader, &mut cache, parsed.clone(), &scope).output;
let executed = exec(&evaluated.template, state.clone()).output;
let layouted = layout(&mut loader, &mut cache, &executed);
let src = std::fs::read_to_string(&path).unwrap();
let tree = Rc::new(parse(&src).output);
let evaluated = eval(&mut loader, &mut cache, &path, tree.clone(), &scope);
let executed = exec(&evaluated.output.template, state.clone());
let layouted = layout(&mut loader, &mut cache, &executed.output);
// Bench!
bench!("parse": parse(&src));
bench!("eval": eval(&mut loader, &mut cache, parsed.clone(), &scope));
bench!("exec": exec(&evaluated.template, state.clone()));
bench!("layout": layout(&mut loader, &mut cache, &executed));
bench!("typeset": typeset(&mut loader, &mut cache, &src, &scope, state.clone()));
bench!("eval": eval(&mut loader, &mut cache, &path, tree.clone(), &scope));
bench!("exec": exec(&evaluated.output.template, state.clone()));
bench!("layout": layout(&mut loader, &mut cache, &executed.output));
bench!("typeset": typeset(&mut loader, &mut cache, &path, &src, &scope, state.clone()));
bench!("pdf": pdf(&cache, &layouted));
}
}

View File

@ -10,27 +10,34 @@ pub use capture::*;
pub use scope::*;
pub use value::*;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use crate::cache::Cache;
use crate::color::Color;
use crate::diag::{Diag, DiagSet, Pass};
use crate::geom::{Angle, Length, Relative};
use crate::loading::Loader;
use crate::loading::{FileHash, Loader};
use crate::parse::parse;
use crate::syntax::visit::Visit;
use crate::syntax::*;
/// Evaluated a parsed source file into a module.
///
/// The `path` should point to the source file for the `tree` and is used to
/// resolve relative path names.
///
/// The `scope` consists of the base definitions that are present from the
/// beginning (typically, the standard library).
pub fn eval(
loader: &mut dyn Loader,
cache: &mut Cache,
path: &Path,
tree: Rc<Tree>,
base: &Scope,
) -> Pass<Module> {
let mut ctx = EvalContext::new(loader, cache, base);
let mut ctx = EvalContext::new(loader, cache, path, base);
let map = tree.eval(&mut ctx);
let module = Module {
scope: ctx.scopes.top,
@ -58,6 +65,12 @@ pub struct EvalContext<'a> {
pub scopes: Scopes<'a>,
/// Evaluation diagnostics.
pub diags: DiagSet,
/// The stack of imported files that led to evaluation of the current file.
pub route: Vec<FileHash>,
/// The location of the currently evaluated file.
pub path: PathBuf,
/// A map of loaded module.
pub modules: HashMap<FileHash, Module>,
}
impl<'a> EvalContext<'a> {
@ -65,20 +78,116 @@ impl<'a> EvalContext<'a> {
pub fn new(
loader: &'a mut dyn Loader,
cache: &'a mut Cache,
path: &Path,
base: &'a Scope,
) -> Self {
let mut route = vec![];
if let Some(hash) = loader.resolve(path) {
route.push(hash);
}
Self {
loader,
cache,
scopes: Scopes::with_base(base),
scopes: Scopes::with_base(Some(base)),
diags: DiagSet::new(),
route,
path: path.to_owned(),
modules: HashMap::new(),
}
}
/// Resolve a path relative to the current file.
///
/// Generates an error if the file is not found.
pub fn resolve(&mut self, path: &str, span: Span) -> Option<(PathBuf, FileHash)> {
let dir = self.path.parent().expect("location is a file");
let path = dir.join(path);
match self.loader.resolve(&path) {
Some(hash) => Some((path, hash)),
None => {
self.diag(error!(span, "file not found"));
None
}
}
}
/// Process an import of a module relative to the current location.
pub fn import(&mut self, path: &str, span: Span) -> Option<FileHash> {
let (resolved, hash) = self.resolve(path, span)?;
// Prevent cycling importing.
if self.route.contains(&hash) {
self.diag(error!(span, "cyclic import"));
return None;
}
if self.modules.get(&hash).is_some() {
return Some(hash);
}
let buffer = self.loader.load_file(&resolved).or_else(|| {
self.diag(error!(span, "failed to load file"));
None
})?;
let string = std::str::from_utf8(&buffer).ok().or_else(|| {
self.diag(error!(span, "file is not valid utf-8"));
None
})?;
// Prepare the new context.
self.route.push(hash);
let new_scopes = Scopes::with_base(self.scopes.base);
let old_scopes = std::mem::replace(&mut self.scopes, new_scopes);
// Evaluate the module.
let tree = Rc::new(parse(string).output);
let map = tree.eval(self);
// Restore the old context.
let new_scopes = std::mem::replace(&mut self.scopes, old_scopes);
self.route.pop();
self.modules.insert(hash, Module {
scope: new_scopes.top,
template: vec![TemplateNode::Tree { tree, map }],
});
Some(hash)
}
/// Add a diagnostic.
pub fn diag(&mut self, diag: Diag) {
self.diags.insert(diag);
}
/// Cast a value to a type and diagnose a possible error / warning.
pub fn cast<T>(&mut self, value: Value, span: Span) -> Option<T>
where
T: Cast<Value>,
{
if value == Value::Error {
return None;
}
match T::cast(value) {
CastResult::Ok(t) => Some(t),
CastResult::Warn(t, m) => {
self.diag(warning!(span, "{}", m));
Some(t)
}
CastResult::Err(value) => {
self.diag(error!(
span,
"expected {}, found {}",
T::TYPE_NAME,
value.type_name(),
));
None
}
}
}
}
/// Evaluate an expression.
@ -349,24 +458,14 @@ impl Eval for CallExpr {
fn eval(&self, ctx: &mut EvalContext) -> Self::Output {
let callee = self.callee.eval(ctx);
if let Value::Func(func) = callee {
let func = func.clone();
if let Some(func) = ctx.cast::<FuncValue>(callee, self.callee.span()) {
let mut args = self.args.eval(ctx);
let returned = func(ctx, &mut args);
args.finish(ctx);
return returned;
} else if callee != Value::Error {
ctx.diag(error!(
self.callee.span(),
"expected function, found {}",
callee.type_name(),
));
returned
} else {
Value::Error
}
Value::Error
}
}
@ -449,7 +548,7 @@ impl Eval for IfExpr {
fn eval(&self, ctx: &mut EvalContext) -> Self::Output {
let condition = self.condition.eval(ctx);
if let Value::Bool(condition) = condition {
if let Some(condition) = ctx.cast(condition, self.condition.span()) {
if condition {
self.if_body.eval(ctx)
} else if let Some(else_body) = &self.else_body {
@ -458,13 +557,6 @@ impl Eval for IfExpr {
Value::None
}
} else {
if condition != Value::Error {
ctx.diag(error!(
self.condition.span(),
"expected boolean, found {}",
condition.type_name(),
));
}
Value::Error
}
}
@ -477,7 +569,7 @@ impl Eval for WhileExpr {
let mut output = vec![];
loop {
let condition = self.condition.eval(ctx);
if let Value::Bool(condition) = condition {
if let Some(condition) = ctx.cast(condition, self.condition.span()) {
if condition {
match self.body.eval(ctx) {
Value::Template(v) => output.extend(v),
@ -489,13 +581,6 @@ impl Eval for WhileExpr {
return Value::Template(output);
}
} else {
if condition != Value::Error {
ctx.diag(error!(
self.condition.span(),
"expected boolean, found {}",
condition.type_name(),
));
}
return Value::Error;
}
}
@ -571,15 +656,54 @@ impl Eval for ForExpr {
impl Eval for ImportExpr {
type Output = Value;
fn eval(&self, _: &mut EvalContext) -> Self::Output {
todo!()
fn eval(&self, ctx: &mut EvalContext) -> Self::Output {
let span = self.path.span();
let path = self.path.eval(ctx);
if let Some(path) = ctx.cast::<String>(path, span) {
if let Some(hash) = ctx.import(&path, span) {
let mut module = &ctx.modules[&hash];
match &self.imports {
Imports::Wildcard => {
for (var, slot) in module.scope.iter() {
let value = slot.borrow().clone();
ctx.scopes.def_mut(var, value);
}
}
Imports::Idents(idents) => {
for ident in idents {
if let Some(slot) = module.scope.get(&ident) {
let value = slot.borrow().clone();
ctx.scopes.def_mut(ident.as_str(), value);
} else {
ctx.diag(error!(ident.span, "unresolved import"));
module = &ctx.modules[&hash];
}
}
}
}
return Value::None;
}
}
Value::Error
}
}
impl Eval for IncludeExpr {
type Output = Value;
fn eval(&self, _: &mut EvalContext) -> Self::Output {
todo!()
fn eval(&self, ctx: &mut EvalContext) -> Self::Output {
let span = self.path.span();
let path = self.path.eval(ctx);
if let Some(path) = ctx.cast::<String>(path, span) {
if let Some(hash) = ctx.import(&path, span) {
return Value::Template(ctx.modules[&hash].template.clone());
}
}
Value::Error
}
}

View File

@ -31,12 +31,8 @@ impl<'a> Scopes<'a> {
}
/// Create a new hierarchy of scopes with a base scope.
pub fn with_base(base: &'a Scope) -> Self {
Self {
top: Scope::new(),
scopes: vec![],
base: Some(base),
}
pub fn with_base(base: Option<&'a Scope>) -> Self {
Self { top: Scope::new(), scopes: vec![], base }
}
/// Enter a new scope.
@ -131,6 +127,11 @@ impl Scope {
pub fn get(&self, var: &str) -> Option<&Slot> {
self.values.get(var)
}
/// Iterate over all definitions.
pub fn iter(&self) -> impl Iterator<Item = (&str, &Slot)> {
self.values.iter().map(|(k, v)| (k.as_str(), v))
}
}
impl Debug for Scope {

View File

@ -3,12 +3,13 @@
use std::collections::{hash_map::Entry, HashMap};
use std::fmt::{self, Debug, Formatter};
use std::io::Cursor;
use std::path::Path;
use image::io::Reader as ImageReader;
use image::{DynamicImage, GenericImageView, ImageFormat};
use serde::{Deserialize, Serialize};
use crate::loading::Loader;
use crate::loading::{FileHash, Loader};
/// A loaded image.
pub struct Image {
@ -56,8 +57,8 @@ impl Debug for Image {
pub struct ImageCache {
/// Loaded images indexed by [`ImageId`].
images: Vec<Image>,
/// Maps from paths to loaded images.
paths: HashMap<String, ImageId>,
/// Maps from file hashes to ids of decoded images.
map: HashMap<FileHash, ImageId>,
/// Callback for loaded images.
on_load: Option<Box<dyn Fn(ImageId, &Image)>>,
}
@ -67,14 +68,14 @@ impl ImageCache {
pub fn new() -> Self {
Self {
images: vec![],
paths: HashMap::new(),
map: HashMap::new(),
on_load: None,
}
}
/// Load and decode an image file from a path.
pub fn load(&mut self, loader: &mut dyn Loader, path: &str) -> Option<ImageId> {
Some(match self.paths.entry(path.to_string()) {
pub fn load(&mut self, loader: &mut dyn Loader, path: &Path) -> Option<ImageId> {
Some(match self.map.entry(loader.resolve(path)?) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let buffer = loader.load_file(path)?;

View File

@ -22,7 +22,6 @@ use std::fmt::{self, Debug, Formatter};
use std::hash::{Hash, Hasher};
use decorum::N64;
use fxhash::FxHasher64;
use crate::cache::Cache;
use crate::geom::*;
@ -81,12 +80,7 @@ impl AnyNode {
where
T: Layout + Debug + Clone + PartialEq + Hash + 'static,
{
let hash = {
let mut state = FxHasher64::default();
node.hash(&mut state);
state.finish()
};
let hash = fxhash::hash64(&node);
Self { node: Box::new(node), hash }
}
}

View File

@ -48,6 +48,7 @@ pub mod pretty;
pub mod syntax;
pub mod util;
use std::path::Path;
use std::rc::Rc;
use crate::cache::Cache;
@ -61,12 +62,13 @@ use crate::loading::Loader;
pub fn typeset(
loader: &mut dyn Loader,
cache: &mut Cache,
path: &Path,
src: &str,
base: &Scope,
state: State,
) -> Pass<Vec<Frame>> {
let parsed = parse::parse(src);
let evaluated = eval::eval(loader, cache, Rc::new(parsed.output), base);
let evaluated = eval::eval(loader, cache, path, Rc::new(parsed.output), base);
let executed = exec::exec(&evaluated.output.template, state);
let layouted = layout::layout(loader, cache, &executed.output);

View File

@ -20,12 +20,14 @@ pub fn image(ctx: &mut EvalContext, args: &mut FuncArgs) -> Value {
let mut node = None;
if let Some(path) = &path {
if let Some(id) = ctx.cache.image.load(ctx.loader, &path.v) {
let img = ctx.cache.image.get(id);
let dimensions = img.buf.dimensions();
node = Some(ImageNode { id, dimensions, width, height });
} else {
ctx.diag(error!(path.span, "failed to load image"));
if let Some((resolved, _)) = ctx.resolve(&path.v, path.span) {
if let Some(id) = ctx.cache.image.load(ctx.loader, &resolved) {
let img = ctx.cache.image.get(id);
let dimensions = img.buf.dimensions();
node = Some(ImageNode { id, dimensions, width, height });
} else {
ctx.diag(error!(path.span, "failed to load image"));
}
}
}

View File

@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
use ttf_parser::{name_id, Face};
use walkdir::WalkDir;
use super::{Buffer, Loader};
use super::{Buffer, FileHash, Loader};
use crate::font::{FaceInfo, FontStretch, FontStyle, FontVariant, FontWeight};
/// Loads fonts and images from the local file system.
@ -25,7 +25,7 @@ pub struct FsLoader {
/// Maps from paths to loaded file buffers. When the buffer is `None` the file
/// does not exist or couldn't be read.
type FileCache = HashMap<PathBuf, Option<Buffer>>;
type FileCache = HashMap<FileHash, Buffer>;
impl FsLoader {
/// Create a new loader without any fonts.
@ -167,24 +167,32 @@ impl Loader for FsLoader {
&self.faces
}
fn resolve(&self, path: &Path) -> Option<FileHash> {
hash(path)
}
fn load_face(&mut self, idx: usize) -> Option<Buffer> {
load(&mut self.cache, &self.files[idx])
}
fn load_file(&mut self, path: &str) -> Option<Buffer> {
load(&mut self.cache, Path::new(path))
fn load_file(&mut self, path: &Path) -> Option<Buffer> {
load(&mut self.cache, path)
}
}
/// Load from the file system using a cache.
fn load(cache: &mut FileCache, path: &Path) -> Option<Buffer> {
match cache.entry(path.to_owned()) {
Some(match cache.entry(hash(path)?) {
Entry::Occupied(entry) => entry.get().clone(),
Entry::Vacant(entry) => {
let buffer = std::fs::read(path).ok().map(Rc::new);
entry.insert(buffer).clone()
let buffer = std::fs::read(path).ok()?;
entry.insert(Rc::new(buffer)).clone()
}
}
})
}
fn hash(path: &Path) -> Option<FileHash> {
path.canonicalize().ok().map(|p| FileHash(fxhash::hash64(&p)))
}
#[cfg(test)]

View File

@ -6,6 +6,7 @@ mod fs;
#[cfg(feature = "fs")]
pub use fs::*;
use std::path::Path;
use std::rc::Rc;
use crate::font::FaceInfo;
@ -18,13 +19,22 @@ pub trait Loader {
/// Descriptions of all font faces this loader serves.
fn faces(&self) -> &[FaceInfo];
/// Resolve a hash that is the same for all paths pointing to the same file.
///
/// Should return `None` if the file does not exist.
fn resolve(&self, path: &Path) -> Option<FileHash>;
/// Load the font face with the given index in [`faces()`](Self::faces).
fn load_face(&mut self, idx: usize) -> Option<Buffer>;
/// Load a file from a path.
fn load_file(&mut self, path: &str) -> Option<Buffer>;
fn load_file(&mut self, path: &Path) -> Option<Buffer>;
}
/// A hash that must be the same for all paths pointing to the same file.
#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
pub struct FileHash(pub u64);
/// A loader which serves nothing.
pub struct BlankLoader;
@ -33,11 +43,15 @@ impl Loader for BlankLoader {
&[]
}
fn resolve(&self, _: &Path) -> Option<FileHash> {
None
}
fn load_face(&mut self, _: usize) -> Option<Buffer> {
None
}
fn load_file(&mut self, _: &str) -> Option<Buffer> {
fn load_file(&mut self, _: &Path) -> Option<Buffer> {
None
}
}

View File

@ -3,37 +3,53 @@ use std::path::{Path, PathBuf};
use anyhow::{anyhow, bail, Context};
use typst::loading::Loader;
fn main() -> anyhow::Result<()> {
let args: Vec<_> = std::env::args().collect();
if args.len() < 2 || args.len() > 3 {
println!("Usage: typst src.typ [out.pdf]");
println!("usage: typst src.typ [out.pdf]");
return Ok(());
}
// Create a loader for fonts and files.
let mut loader = typst::loading::FsLoader::new();
loader.search_path("fonts");
loader.search_system();
// Resolve the canonical path because the compiler needs it for module
// loading.
let src_path = Path::new(&args[1]);
// Find out the file name to create the output file.
let name = src_path
.file_name()
.ok_or_else(|| anyhow!("source path is not a file"))?;
let dest_path = if args.len() <= 2 {
let name = src_path
.file_name()
.ok_or_else(|| anyhow!("Source path is not a file."))?;
Path::new(name).with_extension("pdf")
} else {
PathBuf::from(&args[2])
};
if src_path == dest_path {
bail!("Source and destination path are the same.");
// Ensure that the source file is not overwritten.
let src_hash = loader.resolve(&src_path);
let dest_hash = loader.resolve(&dest_path);
if src_hash.is_some() && src_hash == dest_hash {
bail!("source and destination files are the same");
}
let src = fs::read_to_string(src_path).context("Failed to read from source file.")?;
let mut loader = typst::loading::FsLoader::new();
loader.search_path("fonts");
loader.search_system();
// Read the source.
let src = fs::read_to_string(&src_path)
.map_err(|_| anyhow!("failed to read source file"))?;
// Compile.
let mut cache = typst::cache::Cache::new(&loader);
let scope = typst::library::new();
let state = typst::exec::State::default();
let pass = typst::typeset(&mut loader, &mut cache, &src, &scope, state);
let pass = typst::typeset(&mut loader, &mut cache, &src_path, &src, &scope, state);
// Print diagnostics.
let map = typst::parse::LineMap::new(&src);
for diag in pass.diags {
let start = map.location(diag.span.start).unwrap();
@ -48,8 +64,9 @@ fn main() -> anyhow::Result<()> {
);
}
// Export the PDF.
let buffer = typst::export::pdf(&cache, &pass.output);
fs::write(&dest_path, buffer).context("Failed to write PDF file.")?;
fs::write(&dest_path, buffer).context("failed to write PDF file")?;
Ok(())
}

View File

@ -46,4 +46,4 @@ von _v_ zu einem Blatt. Die Höhe des Baumes ist die Höhe der Wurzel.
// The `image` function returns a "template" value of the same type as
// the `[...]` literals.
#align(center, image("res/graph.png", width: 75%))
#align(center, image("../../res/graph.png", width: 75%))

View File

@ -4,35 +4,35 @@
// Test loading different image formats.
// Load an RGBA PNG image.
#image("res/rhino.png")
#image("../../res/rhino.png")
#pagebreak()
// Load an RGB JPEG image.
#image("res/tiger.jpg")
#image("../../res/tiger.jpg")
// Error: 8-29 failed to load image
// Error: 8-29 file not found
#image("path/does/not/exist")
// Error: 8-29 failed to load image
#image("typ/image-error.typ")
// Error: 8-20 failed to load image
#image("./font.typ")
---
// Test configuring the size and fitting behaviour of images.
// Fit to width of page.
#image("res/rhino.png")
#image("../../res/rhino.png")
// Fit to height of page.
#page(height: 40pt, image("res/rhino.png"))
#page(height: 40pt, image("../../res/rhino.png"))
// Set width explicitly.
#image("res/rhino.png", width: 50pt)
#image("../../res/rhino.png", width: 50pt)
// Set height explicitly.
#image("res/rhino.png", height: 50pt)
#image("../../res/rhino.png", height: 50pt)
// Set width and height explicitly and force stretching.
#image("res/rhino.png", width: 25pt, height: 50pt)
#image("../../res/rhino.png", width: 25pt, height: 50pt)
// Make sure the bounding-box of the image is correct.
#align(bottom, right, image("res/tiger.jpg", width: 60pt))
#align(bottom, right, image("../../res/tiger.jpg", width: 60pt))

View File

@ -46,4 +46,4 @@ Lריווח #h(1cm) R
// Test inline object.
#font("Noto Serif Hebrew", "EB Garamond")
#lang("he")
קרנפיםRh#image("res/rhino.png", height: 11pt)inoחיים
קרנפיםRh#image("../../res/rhino.png", height: 11pt)inoחיים

View File

@ -162,7 +162,7 @@ fn test(
}
} else {
let (part_ok, compare_here, part_frames) =
test_part(loader, cache, part, i, compare_ref, lines);
test_part(loader, cache, src_path, part, i, compare_ref, lines);
ok &= part_ok;
compare_ever |= compare_here;
frames.extend(part_frames);
@ -203,6 +203,7 @@ fn test(
fn test_part(
loader: &mut FsLoader,
cache: &mut Cache,
path: &Path,
src: &str,
i: usize,
compare_ref: bool,
@ -223,7 +224,7 @@ fn test_part(
state.page.size = Size::new(Length::pt(120.0), Length::raw(f64::INFINITY));
state.page.margins = Sides::splat(Some(Length::pt(10.0).into()));
let mut pass = typst::typeset(loader, cache, &src, &scope, state);
let mut pass = typst::typeset(loader, cache, path, &src, &scope, state);
if !compare_ref {
pass.output.clear();
}