From bf3b102face0cdccb57754f8740163e90913b0a4 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin Date: Tue, 7 Nov 2023 04:38:53 +0800 Subject: [PATCH] dev: use fontdb to load system fonts --- Cargo.lock | 18 ++- Cargo.toml | 3 + cli/Cargo.toml | 1 + cli/src/main.rs | 1 - compiler/Cargo.toml | 7 +- compiler/src/font/system.rs | 296 ++++++++++++++++++++---------------- compiler/src/system.rs | 10 +- core/src/config/compiler.rs | 8 - core/src/lib.rs | 4 + 9 files changed, 199 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 153183133..4e0ec98f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1075,13 +1075,24 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fontconfig-parser" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "674e258f4b5d2dcd63888c01c68413c51f565e8af99d2f7701c7b81d79ef41c4" +dependencies = [ + "roxmltree", +] + [[package]] name = "fontdb" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "020e203f177c0fb250fb19455a252e838d2bbbce1f80f25ecc42402aafa8cd38" dependencies = [ + "fontconfig-parser", "log", + "memmap2", "slotmap", "tinyvec", "ttf-parser", @@ -1991,9 +2002,9 @@ checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" -version = "0.9.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaba38d7abf1d4cca21cc89e932e542ba2b9258664d2a9ef0e61512039c9375" +checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed" dependencies = [ "libc", ] @@ -4233,19 +4244,20 @@ dependencies = [ "dissimilar", "ecow", "flate2", + "fontdb", "fst", "hex", "indexmap 2.0.2", "instant", "js-sys", "log", - "memmap2", "nohash-hasher", "notify", "once_cell", "parking_lot", "pathdiff", "pollster", + "rayon", "reqwest", "rustc-hash", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6f1c9ad73..a9432ab63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ tokio-tungstenite = "0.20.0" # system dirs = "5" +fontdb = "0.15.0" # "0.15.0" memmap2 = "0.9" notify = "6" path-clean = "1.0.1" @@ -223,3 +224,5 @@ hayagriva = { git = "https://github.com/Myriad-Dreamin/hayagriva.git", branch = # typst-syntax = { path = "../typst/crates/typst-syntax" } # typst-ide = { path = "../typst/crates/typst-ide" } # hayagriva = { path = "../hayagriva" } + +# fontdb = { path = "../fontdb" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 17c1dfec5..4d312dc90 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -41,6 +41,7 @@ human-panic.workspace = true typst-ts-core.workspace = true typst-ts-compiler = { workspace = true, default-features = false, features = [ "system", + # "lazy-fontdb", "dynamic-layout", ] } diff --git a/cli/src/main.rs b/cli/src/main.rs index 910f4a850..da4bdf320 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -194,7 +194,6 @@ fn measure_fonts(args: MeasureFontsArgs) -> ! { let world = TypstSystemWorld::new(CompileOpts { root_dir: root_path, - font_profile_paths, font_paths: args.font.paths, font_profile_cache_path: args.output.clone(), no_system_fonts: args.no_system_fonts, diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index 6b82a8e27..e67418e5d 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -24,18 +24,20 @@ flate2.workspace = true ecow.workspace = true instant.workspace = true strum.workspace = true +rayon.workspace = true serde.workspace = true serde_json.workspace = true serde-wasm-bindgen = { workspace = true, optional = true } -memmap2 = { workspace = true, optional = true } dirs = { workspace = true, optional = true } walkdir = { workspace = true, optional = true } notify = { workspace = true, optional = true } tokio = { workspace = true, optional = true } pollster = { workspace = true, optional = true } log = { workspace = true, optional = true } +fontdb = { workspace = true, optional = true } + chrono = { workspace = true } base64.workspace = true rustc-hash.workspace = true @@ -85,12 +87,13 @@ serde.workspace = true [features] cjk = [] emoji = [] +lazy-fontdb = [] system-compile = [ - "dep:memmap2", "dep:dirs", "dep:walkdir", "dep:notify", "dep:log", + "dep:fontdb", ] system-watch = ["dep:notify", "dep:tokio"] system = ["system-compile", "system-watch"] diff --git a/compiler/src/font/system.rs b/compiler/src/font/system.rs index ded78b3f5..47956879c 100644 --- a/compiler/src/font/system.rs +++ b/compiler/src/font/system.rs @@ -5,32 +5,24 @@ use std::{ sync::{Arc, Mutex}, }; -use memmap2::Mmap; +use fontdb::Database; use sha2::{Digest, Sha256}; -use typst::font::{FontBook, FontInfo}; -use walkdir::WalkDir; +use typst::{ + diag::{FileError, FileResult}, + font::{FontBook, FontInfo}, +}; use typst_ts_core::{ build_info, font::{ - get_font_coverage_hash, BufferFontLoader, FontInfoItem, FontProfile, FontProfileItem, - FontResolverImpl, LazyBufferFontLoader, PartialFontBook, + BufferFontLoader, FontProfile, FontProfileItem, FontResolverImpl, LazyBufferFontLoader, + PartialFontBook, }, Bytes, FontSlot, }; use crate::vfs::system::LazyFile; -fn is_font_file_by_name(path: &Path) -> bool { - matches!( - path.extension().map(|s| { - let chk = |n| s.eq_ignore_ascii_case(n); - chk("ttf") || chk("otf") || chk("ttc") || chk("otc") - }), - Some(true), - ) -} - #[derive(Default)] struct FontProfileRebuilder { path_items: HashMap, @@ -40,7 +32,8 @@ struct FontProfileRebuilder { impl FontProfileRebuilder { /// Index the fonts in the file at the given path. - pub fn search_file(&mut self, path: impl AsRef) -> Option<&FontProfileItem> { + #[allow(dead_code)] + fn search_file(&mut self, path: impl AsRef) -> Option<&FontProfileItem> { let path = path.as_ref().canonicalize().unwrap(); if let Some(item) = self.path_items.get(&path) { return Some(item); @@ -63,17 +56,17 @@ impl FontProfileRebuilder { // println!("searched font: {:?}", path); - if let Ok(mmap) = unsafe { Mmap::map(&file) } { - for (i, info) in FontInfo::iter(&mmap).enumerate() { - let coverage_hash = get_font_coverage_hash(&info.coverage); - let mut ff = FontInfoItem::new(info); - ff.set_coverage_hash(coverage_hash); - if i != 0 { - ff.set_index(i as u32); - } - profile_item.add_info(ff); - } - } + // if let Ok(mmap) = unsafe { Mmap::map(&file) } { + // for (i, info) in FontInfo::iter(&mmap).enumerate() { + // let coverage_hash = get_font_coverage_hash(&info.coverage); + // let mut ff = FontInfoItem::new(info); + // ff.set_coverage_hash(coverage_hash); + // if i != 0 { + // ff.set_index(i as u32); + // } + // profile_item.add_info(ff); + // } + // } self.profile.items.push(profile_item); return self.profile.items.last(); @@ -85,6 +78,8 @@ impl FontProfileRebuilder { /// Searches for fonts. pub struct SystemFontSearcher { + db: Database, + pub book: FontBook, pub fonts: Vec, profile_rebuilder: FontProfileRebuilder, @@ -96,8 +91,10 @@ impl SystemFontSearcher { let mut profile_rebuilder = FontProfileRebuilder::default(); profile_rebuilder.profile.version = "v1beta".to_owned(); profile_rebuilder.profile.build_info = build_info::VERSION.to_string(); + let db = Database::new(); Self { + db, book: FontBook::new(), fonts: vec![], profile_rebuilder, @@ -142,8 +139,130 @@ impl SystemFontSearcher { // println!("profile_rebuilder init took {:?}", end - begin); } + #[cfg(feature = "lazy-fontdb")] + pub fn flush(&mut self) { + use rayon::prelude::*; + self.db + .lazy_faces() + .enumerate() + .par_bridge() + .flat_map(|(_idx, face)| { + let path = match face.path() { + Some(path) => path, + None => return None, + }; + + #[derive(std::hash::Hash)] + struct CacheStateKey { + path: PathBuf, + index: u32, + } + + #[derive(serde::Serialize, serde::Deserialize)] + struct CacheStateValue { + info: Option, + mtime: u64, + } + + let cache_state_key = CacheStateKey { + path: path.to_owned(), + index: face.index(), + }; + let cache_state_key = typst_ts_core::hash::hash128(&cache_state_key); + let cache_state_path = dirs::cache_dir() + .unwrap_or_else(std::env::temp_dir) + .join("typst") + .join("fonts/v1") + .join(format!("{:x}.json", cache_state_key)); + // println!("cache_state: {:?}", cache_state_path); + let cache_state = std::fs::read_to_string(&cache_state_path).ok(); + let cache_state: Option = cache_state + .as_ref() + .and_then(|s| serde_json::from_str(s).ok()); + + let mtime = path + .metadata() + .ok() + .and_then(|m| m.modified().ok()) + .map(|m| m.duration_since(std::time::UNIX_EPOCH).unwrap().as_micros() as u64) + .unwrap_or_default(); + + let cache_state = cache_state.filter(|cache_state| cache_state.mtime == mtime); + + let info = match cache_state { + Some(cache_state) => cache_state.info, + None => { + let info = face + .with_data(|data| FontInfo::new(data, face.index())) + .expect("database must contain this font"); + std::fs::create_dir_all(cache_state_path.parent().unwrap()).unwrap(); + + let info = CacheStateValue { info, mtime }; + + std::fs::write(&cache_state_path, serde_json::to_string(&info).unwrap()) + .unwrap(); + info.info + } + }; + + // println!("searched font: {idx} {:?}", path); + + Some(( + info?, + FontSlot::new_boxed(LazyBufferFontLoader::new( + LazyFile::new(path.to_owned()), + face.index(), + )), + )) + }) + .collect::>() + .into_iter() + .for_each(|(info, font)| { + self.book.push(info); + self.fonts.push(font); + }); + + self.db = Database::new(); + } + + #[cfg(not(feature = "lazy-fontdb"))] + pub fn flush(&mut self) { + use fontdb::Source; + + for (_idx, face) in self.db.faces().enumerate() { + let path = match &face.source { + Source::File(path) | Source::SharedFile(path, _) => path, + // We never add binary sources to the database, so there + // shouln't be any. + Source::Binary(_) => unreachable!(), + }; + + let info = self + .db + .with_face_data(face.id, FontInfo::new) + .expect("database must contain this font"); + + // println!("searched font: {idx} {:?}", path); + + if let Some(info) = info { + self.book.push(info); + self.fonts + .push(FontSlot::new_boxed(LazyBufferFontLoader::new( + LazyFile::new(path.clone()), + face.index, + ))); + } + } + + self.db = Database::new(); + } + /// Add an in-memory font. pub fn add_memory_font(&mut self, data: Bytes) { + if !self.db.is_empty() { + panic!("dirty font search state, please flush the searcher before adding memory fonts"); + } + for (index, info) in FontInfo::iter(&data).enumerate() { self.book.push(info.clone()); self.fonts.push(FontSlot::new_boxed(BufferFontLoader { @@ -153,117 +272,20 @@ impl SystemFontSearcher { } } - /// Add fonts that in vanilla style. - pub fn search_vanilla(&mut self) { - let program_dir = std::env::current_exe().unwrap(); - self.search_vanilla_one(&program_dir); - self.search_vanilla_one(std::env::current_dir().unwrap().as_path()); - } - - fn search_vanilla_one(&mut self, program_dir: &Path) { - let mut program_dir = program_dir.parent(); - while let Some(dir) = program_dir { - let path = dir; - for vanilla_dir in &["fonts", "assets/fonts", "asset/fonts"] { - let fonts_dir = path.join(vanilla_dir); - if fonts_dir.exists() { - self.search_dir(&fonts_dir); - } - } - program_dir = path.parent(); - } - } - pub fn search_system(&mut self) { - let font_paths = { - // Search for fonts in the linux system font directories. - #[cfg(all(unix, not(target_os = "macos")))] - { - let mut font_paths = ["/usr/share/fonts", "/usr/local/share/fonts"] - .iter() - .map(PathBuf::from) - .collect::>(); - - if let Some(dir) = dirs::font_dir() { - font_paths.push(dir); - } - - font_paths - } - // Search for fonts in the macOS system font directories. - #[cfg(target_os = "macos")] - { - let mut font_paths = [ - "/Library/Fonts", - "/Network/Library/Fonts", - "/System/Library/Fonts", - ] - .iter() - .map(PathBuf::from) - .collect::>(); - - if let Some(dir) = dirs::font_dir() { - font_paths.push(dir); - } - - font_paths - } - // Search for fonts in the Windows system font directories. - #[cfg(windows)] - { - let mut font_paths = vec![]; - let windir = std::env::var("WINDIR").unwrap_or_else(|_| "C:\\Windows".to_string()); - - font_paths.push(PathBuf::from(windir).join("Fonts")); - - if let Some(roaming) = dirs::config_dir() { - font_paths.push(roaming.join("Microsoft\\Windows\\Fonts")); - } - if let Some(local) = dirs::cache_dir() { - font_paths.push(local.join("Microsoft\\Windows\\Fonts")); - } - - font_paths - } - }; - - for dir in font_paths { - self.search_dir(dir); - } + self.db.load_system_fonts(); } /// Search for all fonts in a directory recursively. pub fn search_dir(&mut self, path: impl AsRef) { - let entries = WalkDir::new(path) - .follow_links(true) - .sort_by(|a, b| a.file_name().cmp(b.file_name())) - .into_iter() - // todo: error handling - .filter_map(|e| e.ok()); - - for entry in entries { - let path = entry.path(); - if is_font_file_by_name(path) { - self.search_file(path); - } - } + self.db.load_fonts_dir(path); } /// Index the fonts in the file at the given path. - pub fn search_file(&mut self, path: impl AsRef) { - let profile_item = match self.profile_rebuilder.search_file(path.as_ref()) { - Some(profile_item) => profile_item, - None => return, - }; - - for info in profile_item.info.iter() { - self.book.push(info.info.clone()); - self.fonts - .push(FontSlot::new_boxed(LazyBufferFontLoader::new( - LazyFile::new(path.as_ref().to_owned()), - info.index().unwrap_or_default(), - ))); - } + pub fn search_file(&mut self, path: impl AsRef) -> FileResult<()> { + self.db + .load_font_file(path.as_ref()) + .map_err(|e| FileError::from_io(e, path.as_ref())) } } @@ -275,6 +297,20 @@ impl Default for SystemFontSearcher { impl From for FontResolverImpl { fn from(searcher: SystemFontSearcher) -> Self { + // let profile_item = match + // self.profile_rebuilder.search_file(path.as_ref()) { + // Some(profile_item) => profile_item, + // None => return, + // }; + + // for info in profile_item.info.iter() { + // self.book.push(info.info.clone()); + // self.fonts + // .push(FontSlot::new_boxed(LazyBufferFontLoader::new( + // LazyFile::new(path.as_ref().to_owned()), + // info.index().unwrap_or_default(), + // ))); + // } FontResolverImpl::new( searcher.book, Arc::new(Mutex::new(PartialFontBook::default())), diff --git a/compiler/src/system.rs b/compiler/src/system.rs index dfca7599b..46292cd0d 100644 --- a/compiler/src/system.rs +++ b/compiler/src/system.rs @@ -54,13 +54,17 @@ impl TypstSystemWorld { if path.is_dir() { searcher.search_dir(&path); } else { - searcher.search_file(&path); + let _ = searcher.search_file(&path); } } // Source2: add the fonts from system paths. if !opts.no_system_fonts { searcher.search_system(); } + + // flush source1 and source2 before adding source3 + searcher.flush(); + // Source3: add the fonts in memory. for font_data in opts.with_embedded_fonts { searcher.add_memory_font(match font_data { @@ -68,10 +72,6 @@ impl TypstSystemWorld { Cow::Owned(data) => Bytes::from(data), }); } - // Source4: add the fonts from the profile cache. - for profile_path in opts.font_profile_paths { - searcher.add_profile_by_path(&profile_path); - } Ok(searcher.into()) } diff --git a/core/src/config/compiler.rs b/core/src/config/compiler.rs index fd9b1ddb5..b7678ca78 100644 --- a/core/src/config/compiler.rs +++ b/core/src/config/compiler.rs @@ -15,10 +15,6 @@ pub struct CompileOpts { /// Path to entry pub entry: PathBuf, - /// Path to font profiles - #[serde(rename = "fontProfilePaths")] - pub font_profile_paths: Vec, - /// Path to font profile for cache #[serde(rename = "fontProfileCachePath")] pub font_profile_cache_path: PathBuf, @@ -31,10 +27,6 @@ pub struct CompileOpts { #[serde(rename = "noSystemFonts")] pub no_system_fonts: bool, - /// Exclude vanilla font paths - #[serde(rename = "noVanillaFonts")] - pub no_vanilla_fonts: bool, - /// Include embedded fonts #[serde(rename = "withEmbeddedFonts")] #[serde_as(as = "Vec")] diff --git a/core/src/lib.rs b/core/src/lib.rs index d57683421..3379f7a8e 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,4 +1,8 @@ // Core type system/concepts of typst-ts. +// #![warn(missing_docs)] +// #![warn(missing_debug_implementations)] +// #![warn(missing_copy_implementations)] + pub(crate) mod concepts; pub use concepts::*; pub mod error;