diff --git a/packages/compiler/Cargo.toml b/packages/compiler/Cargo.toml index 780184fe9..b7a329dc4 100644 --- a/packages/compiler/Cargo.toml +++ b/packages/compiler/Cargo.toml @@ -23,6 +23,7 @@ comemo = "0.2" once_cell = "1.17.1" siphasher = "0.3.10" elsa = "1.8.0" +serde-wasm-bindgen = "^0.5" # Everything to do with wasm wasm-bindgen = { version = "^0.2" } diff --git a/packages/compiler/src/lib.rs b/packages/compiler/src/lib.rs index 368ec8ed0..208299594 100644 --- a/packages/compiler/src/lib.rs +++ b/packages/compiler/src/lib.rs @@ -1,8 +1,9 @@ use std::sync::{Arc, Mutex}; +use js_sys::{JsString, Uint8Array}; use typst_ts_compiler::font::web::BrowserFontSearcher; pub use typst_ts_compiler::*; -use typst_ts_core::DocumentExporter; +use typst_ts_core::{cache::FontInfoCache, DocumentExporter}; use wasm_bindgen::prelude::*; use crate::utils::console_log; @@ -27,6 +28,11 @@ impl TypstCompiler { } } +#[wasm_bindgen] +pub fn get_font_info(buffer: Uint8Array) -> JsValue { + serde_wasm_bindgen::to_value(&FontInfoCache::from_data(buffer.to_vec().as_slice())).unwrap() +} + #[wasm_bindgen] impl TypstCompiler { pub fn reset(&mut self) { @@ -54,6 +60,26 @@ impl TypstCompiler { } } + pub fn modify_font_data(&mut self, idx: usize, buffer: Uint8Array) { + self.world + .font_resolver + .modify_font_data(idx, buffer.to_vec().into()); + } + + pub fn rebuild(&mut self) { + if self.world.font_resolver.partial_resolved() { + self.world.font_resolver.rebuild(); + } + } + + pub fn get_loaded_fonts(&mut self) -> Vec { + self.world + .font_resolver + .loaded_fonts() + .map(|s| format!("<{}, {:?}>", s.0, s.1).into()) + .collect() + } + pub fn get_ast(&mut self) -> Result { let data = Arc::new(Mutex::new(vec![])); @@ -64,9 +90,17 @@ impl TypstCompiler { Ok(()) })); + // let artifact = Arc::new(Mutex::new(None)); + // compile and export document typst::compile(&self.world) - .and_then(|output| ast_exporter.export(&self.world, &output)) + .and_then(|output| { + // let mut artifact = artifact.lock().unwrap(); + // artifact = Some(Artifact::from(&output)); + // drop(artifact); + + ast_exporter.export(&self.world, &output) + }) .unwrap(); let converted = ansi_to_html::convert_escaped( String::from_utf8(data.lock().unwrap().clone()) diff --git a/packages/typst.ts/examples/compiler.html b/packages/typst.ts/examples/compiler.html index c5ec3cc04..780305db2 100644 --- a/packages/typst.ts/examples/compiler.html +++ b/packages/typst.ts/examples/compiler.html @@ -3,7 +3,7 @@ - + Typst.ts @@ -26,6 +26,23 @@ const terminalContent = document.getElementById('terminal-content'); terminalContent.innerHTML = 'Compiling...'; + const runCompile = async () => { + const data = await fetch('http://localhost:20810/skyzh-cv/main.typ').then(res => + res.text(), + ); + + const begin = performance.now(); + compilerPlugin.addSource('main.typ', data, true); + const ast = await compilerPlugin.getAst(); + const end = performance.now(); + const rounded = Math.round((end - begin) * 1000) / 1000; + + const compileInfo = `--- + Compiled in ${rounded}ms`; + + terminalContent.innerHTML = [compileInfo, ast].join('\n'); + }; + let compilerPlugin = window.TypstCompileModule.createTypstCompiler(); compilerPlugin .init({ @@ -45,44 +62,11 @@ getModule: () => '/node_modules/@myriaddreamin/typst-ts-web-compiler/typst_ts_web_compiler_bg.wasm', }) - .then(async () => { - const data = await fetch('http://localhost:20810/skyzh-cv/main.typ').then(res => - res.text(), - ); - - const begin = performance.now(); - compilerPlugin.addSource('main.typ', data, true); - const ast = await compilerPlugin.getAst(); - const end = performance.now(); - const rounded = Math.round((end - begin) * 1000) / 1000; - - const compileInfo = `--- -Compiled in ${rounded}ms`; - - terminalContent.innerHTML = [compileInfo, ast].join('\n'); - }); - - let rendererPlugin = window.TypstRenderModule.createTypstRenderer(pdfjsLib); - rendererPlugin - .init({ - beforeBuild: [ - window.TypstRenderModule.preloadRemoteFonts([ - 'http://localhost:20811/fonts/LinLibertine_R.ttf', - 'http://localhost:20811/fonts/LinLibertine_RB.ttf', - 'http://localhost:20811/fonts/LinLibertine_RBI.ttf', - 'http://localhost:20811/fonts/LinLibertine_RI.ttf', - 'http://localhost:20811/fonts/NewCMMath-Book.otf', - 'http://localhost:20811/fonts/NewCMMath-Regular.otf', - ]), - window.TypstRenderModule.preloadSystemFonts({ - byFamily: ['Segoe UI Symbol'], - }), - ], - getModule: () => - '/node_modules/@myriaddreamin/typst-ts-renderer/typst_ts_renderer_bg.wasm', - }) .then(() => { - console.log('renderer init'); + document.getElementById('compile-button').addEventListener('click', () => { + runCompile(); + }); + return runCompile(); }); }); @@ -149,6 +133,9 @@
+
+ + 
hello world
diff --git a/packages/typst.ts/package.json b/packages/typst.ts/package.json index 59ff09d5c..3d47e4d02 100644 --- a/packages/typst.ts/package.json +++ b/packages/typst.ts/package.json @@ -39,5 +39,8 @@ "esbuild-plugin-wasm": "^1.0.0", "tslib": "2.4.0", "typescript": "4.7.4" + }, + "dependencies": { + "idb": "^7.1.1" } } diff --git a/packages/typst.ts/src/compiler.ts b/packages/typst.ts/src/compiler.ts index 4b1f3bbfb..fc30164d2 100644 --- a/packages/typst.ts/src/compiler.ts +++ b/packages/typst.ts/src/compiler.ts @@ -1,6 +1,6 @@ // @ts-ignore import typstInit, * as typst from '../../compiler/pkg/typst_ts_web_compiler'; -import { buildComponent } from './init'; +import { buildComponent, globalFontPromises } from './init'; import type { InitOptions, BeforeBuildMark } from './options.init'; import { LazyWasmModule } from './wasm'; @@ -47,10 +47,22 @@ class TypstCompilerDriver { this.compiler = await buildComponent(options, gCompilerModule, typst.TypstCompilerBuilder, {}); } - async loadFont(builder: typst.TypstCompilerBuilder, fontPath: string): Promise { - const response = await fetch(fontPath); - const fontBuffer = new Uint8Array(await response.arrayBuffer()); - await builder.add_raw_font(fontBuffer); + async runSyncCodeUntilStable(execute: () => T): Promise { + do { + console.log(this.compiler.get_loaded_fonts()); + const result = await execute(); + console.log(this.compiler.get_loaded_fonts()); + if (globalFontPromises.length > 0) { + const promises = globalFontPromises.splice(0, globalFontPromises.length); + const callbacks = await Promise.all(promises); + for (const callback of callbacks) { + this.compiler.modify_font_data(callback.idx, new Uint8Array(callback.buffer)); + } + this.compiler.rebuild(); + continue; + } + return result; + } while (true); } async reset(): Promise { @@ -62,7 +74,7 @@ class TypstCompilerDriver { } async getAst(): Promise { - return this.compiler.get_ast(); + return this.runSyncCodeUntilStable(() => this.compiler.get_ast()); } async compile(options: any): Promise { diff --git a/packages/typst.ts/src/init.ts b/packages/typst.ts/src/init.ts index 04ea9688d..51b788b4e 100644 --- a/packages/typst.ts/src/init.ts +++ b/packages/typst.ts/src/init.ts @@ -1,5 +1,7 @@ +import { get_font_info } from '../../compiler/pkg/typst_ts_web_compiler'; import { BeforeBuildMark, InitOptions } from './options.init'; import { LazyWasmModule } from './wasm'; +import * as idb from 'idb'; /** @internal */ export interface TypstCommonBuilder { @@ -23,42 +25,110 @@ interface InitContext { hooks: ComponentBuildHooks; } +/** @internal */ +export const globalFontPromises: Promise<{ buffer: ArrayBuffer; idx: number }>[] = []; + async function addPartialFonts({ builder, hooks }: InitContext): Promise { const t = performance.now(); if ('queryLocalFonts' in window) { const fonts: any[] = await (window as any).queryLocalFonts(); console.log('local fonts count:', fonts.length); - await builder.add_web_fonts(fonts); + + const db = await idb.openDB('typst-ts-store', 1, { + upgrade(db) { + db.createObjectStore('font-information', { + keyPath: 'postscriptName', + }); + }, + }); + + const informations = await Promise.all( + fonts.map(async font => { + const postscriptName = font.postscriptName; + + return (await db.get('font-information', postscriptName))?.info; + }), + ); + + await builder.add_web_fonts( + fonts.map((font, font_idx) => { + let gettingBuffer = false; + let readyBuffer: ArrayBuffer | undefined = undefined; + const fullName = font.fullName; + const postscriptName = font.postscriptName; + + const prev = informations[font_idx]; + if (prev) { + console.log('prev', postscriptName, prev); + } + return { + family: font.family, + style: font.style, + fullName: fullName, + postscriptName: postscriptName, + ref: font, + info: informations[font_idx], + blob: (idx: number) => { + console.log(this, font, idx); + if (readyBuffer) { + return readyBuffer; + } + if (gettingBuffer) { + return; + } + gettingBuffer = true; + globalFontPromises.push( + (async () => { + const blob: Blob = await font.blob(); + const buffer = await blob.arrayBuffer(); + readyBuffer = buffer; + const realFontInfo = get_font_info(new Uint8Array(buffer)); + console.log(realFontInfo); + + db.put('font-information', { + fullName, + postscriptName, + info: realFontInfo, + }); + + return { buffer, idx }; + })(), + ); + }, + }; + }), + ); } const t2 = performance.now(); console.log('addPartialFonts time used:', t2 - t); } -/** @internal */ -async function buildComponentInternal( - options: Partial | undefined, - gModule: LazyWasmModule, - builder: TypstCommonBuilder, - hooks: ComponentBuildHooks, -): Promise { - /// init typst wasm module - if (options?.getModule) { - await gModule.init(options.getModule()); +class ComponentBuilder { + async loadFont(builder: TypstCommonBuilder, fontPath: string): Promise { + const response = await fetch(fontPath); + const fontBuffer = new Uint8Array(await response.arrayBuffer()); + await builder.add_raw_font(fontBuffer); } - /// build typst component - const buildCtx: InitContext = { ref: this, builder, hooks }; + async build( + options: Partial | undefined, + builder: TypstCommonBuilder, + hooks: ComponentBuildHooks, + ): Promise { + /// build typst component + const buildCtx: InitContext = { ref: this, builder, hooks }; - for (const fn of options?.beforeBuild ?? []) { - await fn(undefined as unknown as BeforeBuildMark, buildCtx); - } - addPartialFonts(buildCtx); + for (const fn of options?.beforeBuild ?? []) { + await fn(undefined as unknown as BeforeBuildMark, buildCtx); + } + await addPartialFonts(buildCtx); - const component = await builder.build(); + const component = await builder.build(); - return component; + return component; + } } /** @internal */ @@ -68,10 +138,10 @@ export async function buildComponent( Builder: { new (): TypstCommonBuilder }, hooks: ComponentBuildHooks, ): Promise { - const builder = new Builder(); - try { - return buildComponentInternal(options, gModule, builder, hooks); - } finally { - builder.free(); + /// init typst wasm module + if (options?.getModule) { + await gModule.init(options.getModule()); } + + return await new ComponentBuilder().build(options, new Builder(), hooks); } diff --git a/packages/typst.ts/src/renderer.ts b/packages/typst.ts/src/renderer.ts index 79c94c341..9caa486b4 100644 --- a/packages/typst.ts/src/renderer.ts +++ b/packages/typst.ts/src/renderer.ts @@ -71,12 +71,6 @@ class TypstRendererDriver { constructor(private pdf: typeof pdfjsModule) {} - async loadFont(builder: typst.TypstRendererBuilder, fontPath: string): Promise { - const response = await fetch(fontPath); - const fontBuffer = new Uint8Array(await response.arrayBuffer()); - await builder.add_raw_font(fontBuffer); - } - async init(options?: Partial): Promise { this.renderer = await buildComponent(options, gRendererModule, typst.TypstRendererBuilder, {}); } diff --git a/packages/typst.ts/tsconfig.json b/packages/typst.ts/tsconfig.json index 0f9ee7dd8..8aa9c352c 100644 --- a/packages/typst.ts/tsconfig.json +++ b/packages/typst.ts/tsconfig.json @@ -14,7 +14,7 @@ "strictNullChecks": true, "declaration": true, "stripInternal": true, - "lib": ["ES5", "ES6", "ES7"] + "lib": ["ES5", "ES6", "ES7", "ES2018"] }, "include": ["src/**/*.ts"] } diff --git a/packages/typst.ts/yarn.lock b/packages/typst.ts/yarn.lock index f26520753..a9cb5c141 100644 --- a/packages/typst.ts/yarn.lock +++ b/packages/typst.ts/yarn.lock @@ -154,9 +154,9 @@ integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== "@types/node@^16.11.6": - version "16.18.23" - resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.23.tgz#b6e934fe427eb7081d0015aad070acb3373c3c90" - integrity sha512-XAMpaw1s1+6zM+jn2tmw8MyaRDIJfXxqmIQIS0HfoGYPuf7dUWeiUKopwq13KFX9lEp1+THGtlaaYx39Nxr58g== + version "16.18.31" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.31.tgz#7de39c2b9363f0d95b129cc969fcbf98e870251c" + integrity sha512-KPXltf4z4g517OlVJO9XQ2357CYw7fvuJ3ZuBynjXC5Jos9i+K7LvFb7bUIwtJXSZj0vTp9Q6NJBSQpkwwO8Zw== "@types/pdfjs-dist@^2.10.378": version "2.10.378" @@ -310,7 +310,7 @@ builtin-modules@3.3.0: resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-3.3.0.tgz#cae62812b89801e9656336e46223e030386be7b6" integrity sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw== -canvas@^2.11.0: +canvas@^2.11.2: version "2.11.2" resolved "https://registry.yarnpkg.com/canvas/-/canvas-2.11.2.tgz#553d87b1e0228c7ac0fc72887c3adbac4abbd860" integrity sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw== @@ -376,9 +376,9 @@ emoji-regex@^8.0.0: integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== esbuild-plugin-wasm@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/esbuild-plugin-wasm/-/esbuild-plugin-wasm-1.0.0.tgz#4ca95ac4ae553331d2b0099223f8713a49b6cff2" - integrity sha512-iXIf3hwfqorExG66/eNr3U8JakIZuge70nMNQtinvxbzdljQ/RjvwaBiGPqF/DvuIumUApbe3zj2kqHLVyc7uQ== + version "1.1.0" + resolved "https://registry.yarnpkg.com/esbuild-plugin-wasm/-/esbuild-plugin-wasm-1.1.0.tgz#062c0e62c266e94165c66ebcbb5852a1cdbfd7cd" + integrity sha512-0bQ6+1tUbySSnxzn5jnXHMDvYnT0cN/Wd4Syk8g/sqAIJUg7buTIi22svS3Qz6ssx895NT+TgLPb33xi1OkZig== esbuild@0.17.3: version "0.17.3" @@ -429,9 +429,9 @@ eslint-visitor-keys@^2.0.0: integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== eslint-visitor-keys@^3.3.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz#c7f0f956124ce677047ddbc192a68f999454dedc" - integrity sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ== + version "3.4.1" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz#c22c48f48942d08ca824cc526211ae400478a994" + integrity sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA== esrecurse@^4.3.0: version "4.3.0" @@ -551,6 +551,11 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" +idb@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/idb/-/idb-7.1.1.tgz#d910ded866d32c7ced9befc5bfdf36f572ced72b" + integrity sha512-gchesWBzyvGHRO9W8tzUWFDycow5gwjvFKfyV9FF32Y7F50yZMp7mP+T2mJIWFx49zicqyC4uefHM17o6xKIVQ== + ignore@^5.2.0: version "5.2.4" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" @@ -637,10 +642,10 @@ minipass@^3.0.0: dependencies: yallist "^4.0.0" -minipass@^4.0.0: - version "4.2.8" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-4.2.8.tgz#f0010f64393ecfc1d1ccb5f582bcaf45f48e1a3a" - integrity sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ== +minipass@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" + integrity sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ== minizlib@^2.1.1: version "2.1.2" @@ -666,9 +671,9 @@ nan@^2.17.0: integrity sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ== node-fetch@^2.6.7: - version "2.6.9" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" - integrity sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg== + version "2.6.11" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25" + integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w== dependencies: whatwg-url "^5.0.0" @@ -717,14 +722,14 @@ path2d-polyfill@^2.0.1: integrity sha512-ad/3bsalbbWhmBo0D6FZ4RNMwsLsPpL6gnvhuSaU5Vm7b06Kr5ubSltQQ0T7YKsiJQO+g22zJ4dJKNTXIyOXtA== pdfjs-dist@*: - version "3.5.141" - resolved "https://registry.yarnpkg.com/pdfjs-dist/-/pdfjs-dist-3.5.141.tgz#a98c2d4dd3192592ba7055d3e450ed6e97112f41" - integrity sha512-lYIvyi5grtYOIatsfCifIKwxHeAJ8eHyP22DTdvY4pm0yWVSFQnMafpgCPSw8gaNRDDdcHnBVOkqMsyK8SRxZg== + version "3.6.172" + resolved "https://registry.yarnpkg.com/pdfjs-dist/-/pdfjs-dist-3.6.172.tgz#f9efdfc5e850e1fecfc70b7f6f45c5dc990d8096" + integrity sha512-bfOhCg+S9DXh/ImWhWYTOiq3aVMFSCvzGiBzsIJtdMC71kVWDBw7UXr32xh0y56qc5wMVylIeqV3hBaRsu+e+w== dependencies: path2d-polyfill "^2.0.1" web-streams-polyfill "^3.2.1" optionalDependencies: - canvas "^2.11.0" + canvas "^2.11.2" picomatch@^2.3.1: version "2.3.1" @@ -780,9 +785,9 @@ semver@^6.0.0: integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== semver@^7.3.5, semver@^7.3.7: - version "7.5.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.0.tgz#ed8c5dc8efb6c629c88b23d41dc9bf40c1d96cd0" - integrity sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA== + version "7.5.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.1.tgz#c90c4d631cf74720e46b21c1d37ea07edfab91ec" + integrity sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw== dependencies: lru-cache "^6.0.0" @@ -839,13 +844,13 @@ strip-ansi@^6.0.1: ansi-regex "^5.0.1" tar@^6.1.11: - version "6.1.13" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.13.tgz#46e22529000f612180601a6fe0680e7da508847b" - integrity sha512-jdIBIN6LTIe2jqzay/2vtYLlBHa3JF42ot3h1dW8Q0PaAG4v8rm0cvpVePtau5C6OKXGGcgO9q2AMNSWxiLqKw== + version "6.1.15" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69" + integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A== dependencies: chownr "^2.0.0" fs-minipass "^2.0.0" - minipass "^4.0.0" + minipass "^5.0.0" minizlib "^2.1.1" mkdirp "^1.0.3" yallist "^4.0.0"