From 59066c3ae2cfdc6a81bb6117bc8bd6305d8208c6 Mon Sep 17 00:00:00 2001
From: Myriad-Dreamin <35292584+Myriad-Dreamin@users.noreply.github.com>
Date: Wed, 20 Sep 2023 14:15:07 +0800
Subject: [PATCH] dev(compiler): new font distribute strategy (#362)

* dev(compiler): new font distribute strategy

* build: update submodule index

* fix(cli): specify cjk,emoji fonts
---
 .github/workflows/ci.yaml      | 12 +++---------
 .github/workflows/docker.yaml  |  4 +++-
 .github/workflows/gh_pages.yml |  4 +++-
 .github/workflows/release.yaml | 15 ++++++++++++---
 .gitignore                     |  3 +--
 .gitmodules                    |  8 ++++++++
 README.md                      |  2 +-
 assets/files                   |  1 +
 assets/fonts                   |  1 +
 cli/Cargo.toml                 | 11 ++++++++++-
 compiler/src/system.rs         | 10 +++-------
 docs/download-font-assets.md   | 18 ++++++++++++++++++
 tools/fontctl/src/main.rs      |  9 ---------
 13 files changed, 64 insertions(+), 34 deletions(-)
 create mode 100644 .gitmodules
 create mode 160000 assets/files
 create mode 160000 assets/fonts

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a23a7aed6..2f2019495 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -11,8 +11,10 @@ jobs:
   ci:
     runs-on: ubuntu-latest
     steps:
-      - name: Download Repo
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install Mold
         uses: rui314/setup-mold@v1
       - name: Install Rust
@@ -36,14 +38,6 @@ jobs:
         run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | bash
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3
-      - name: Prepare fonts
-        run: cargo run --profile release-ci --bin typst-ts-fontctl
-      - name: Prepare assets
-        run: |
-          git clone https://github.com/Myriad-Dreamin/typst fuzzers/typst
-          cd fuzzers/typst
-          git reset --hard 20a988e48374675f7d0d05c4822cd297a3293f41
-          cp -r ./assets/files/ ../../assets/files/
       - name: Fmt check
         run: cargo fmt --check --all
       - name: Clippy check
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
index 35516f438..f06b9a52c 100644
--- a/.github/workflows/docker.yaml
+++ b/.github/workflows/docker.yaml
@@ -17,8 +17,10 @@ jobs:
       packages: write
 
     steps:
-      - name: Checkout repository
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
 
       - name: Setup Docker buildx
         uses: docker/setup-buildx-action@v2.5.0
diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml
index 8ea9aedc8..1a2321202 100644
--- a/.github/workflows/gh_pages.yml
+++ b/.github/workflows/gh_pages.yml
@@ -32,8 +32,10 @@ jobs:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
-      - name: Checkout
+      - name: Checkout Repo
         uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Install Mold
         uses: rui314/setup-mold@v1
       - name: Install Rust
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 11159f1c1..81d1de884 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -54,7 +54,10 @@ jobs:
     env:
       target: ${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Run sccache-cache
         uses: mozilla-actions/sccache-action@v0.0.3
       - name: rust toolchain
@@ -104,7 +107,10 @@ jobs:
     env:
       target: ${{ matrix.platform }}-${{ matrix.arch }}
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: ${{ matrix.rust-target }}
@@ -187,7 +193,10 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
         with:
           fetch-depth: 0
       - uses: actions/download-artifact@v2
diff --git a/.gitignore b/.gitignore
index dc1769ea2..05bec28a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,7 +12,6 @@
 
 target/
 out/
-/assets/
-/assets
+/assets/profile
 dist/
 node_modules/
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..ecb716b24
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,8 @@
+[submodule "TypstFontAssets"]
+	path = assets/fonts
+	url = https://github.com/Myriad-Dreamin/typst.git
+	branch = assets-fonts
+[submodule "TypstFileAssets"]
+	path = assets/files
+	url = https://github.com/Myriad-Dreamin/typst.git
+	branch = assets-files
diff --git a/README.md b/README.md
index a47691e60..f0c7a452e 100644
--- a/README.md
+++ b/README.md
@@ -109,7 +109,7 @@ Note: Since we use turborepo for `>=v0.4.0` development, if you are the earlier
 
 ```shell
 # Optional: download the font assets if you haven't done so.
-$ cargo run --bin typst-ts-fontctl
+$ git submodule update --init --recursive .
 # build all of typescript packages
 $ yarn install && npx turbo run build
 # compile typst document for demo
diff --git a/assets/files b/assets/files
new file mode 160000
index 000000000..dcddda130
--- /dev/null
+++ b/assets/files
@@ -0,0 +1 @@
+Subproject commit dcddda130191e5a93af426f5eb8f2fa482790edd
diff --git a/assets/fonts b/assets/fonts
new file mode 160000
index 000000000..0149f1d51
--- /dev/null
+++ b/assets/fonts
@@ -0,0 +1 @@
+Subproject commit 0149f1d5166df2a565d36650da0ba076c022c48e
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 87788766d..8e7f94d2c 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -62,4 +62,13 @@ raster = ["typst-ts-raster-exporter"]
 serde-json = ["typst-ts-serde-exporter", "typst-ts-serde-exporter/json"]
 serde-rmp = ["typst-ts-serde-exporter", "typst-ts-serde-exporter/rmp"]
 svg = ["typst-ts-svg-exporter"]
-default = ["pdf", "raster", "serde-json", "serde-rmp", "svg", "embedded-fonts"]
+default = [
+    "pdf",
+    "raster",
+    "serde-json",
+    "serde-rmp",
+    "svg",
+    "embedded-fonts",
+    "embedded-cjk-fonts",
+    "embedded-emoji-fonts",
+]
diff --git a/compiler/src/system.rs b/compiler/src/system.rs
index ba24b7610..dfca7599b 100644
--- a/compiler/src/system.rs
+++ b/compiler/src/system.rs
@@ -57,22 +57,18 @@ impl TypstSystemWorld {
                 searcher.search_file(&path);
             }
         }
-        // Source2: add the fonts from vanilla paths.
-        if !opts.no_vanilla_fonts {
-            searcher.search_vanilla();
-        }
-        // Source3: add the fonts from system paths.
+        // Source2: add the fonts from system paths.
         if !opts.no_system_fonts {
             searcher.search_system();
         }
-        // Source4: add the fonts in memory.
+        // Source3: add the fonts in memory.
         for font_data in opts.with_embedded_fonts {
             searcher.add_memory_font(match font_data {
                 Cow::Borrowed(data) => Bytes::from_static(data),
                 Cow::Owned(data) => Bytes::from(data),
             });
         }
-        // Source5: add the fonts from the profile cache.
+        // Source4: add the fonts from the profile cache.
         for profile_path in opts.font_profile_paths {
             searcher.add_profile_by_path(&profile_path);
         }
diff --git a/docs/download-font-assets.md b/docs/download-font-assets.md
index 74565af48..ed23ef381 100644
--- a/docs/download-font-assets.md
+++ b/docs/download-font-assets.md
@@ -10,6 +10,24 @@ There are several ways to downloading the font files:
 
 - Download the font files from [Typst repository](https://github.com/typst/typst/tree/main/assets/fonts).
 
+- Download the font files via `git`:
+
+  Please use following command inside of `typst.ts` repo:
+
+  ```shell
+  # init inside typst.ts repository
+  $ git submodule update --init --recursive .
+  # update inside typst.ts repository
+  $ git submodule update --recursive
+  ```
+
+  Please use following command outside of `typst.ts` repo:
+
+  ```shell
+  # outside typst.ts repository
+  $ git clone https://github.com/Myriad-Dreamin/typst/ fonts --single-branch --branch assets-fonts --depth 1
+  ```
+
 - Download font files from our [Release Page](https://github.com/Myriad-Dreamin/typst.ts/releases/tag/v0.1.0).
 
 - Download font files using `tools/fontctl`
diff --git a/tools/fontctl/src/main.rs b/tools/fontctl/src/main.rs
index c2a58d488..33039a5d5 100644
--- a/tools/fontctl/src/main.rs
+++ b/tools/fontctl/src/main.rs
@@ -153,15 +153,6 @@ fn download_fonts(opts: Opts) {
     println!("downloading...");
     repo.exec().unwrap();
 
-    std::fs::create_dir_all("github-pages/assets/fonts").unwrap();
-    for font in FONT_LISTS {
-        std::fs::copy(
-            format!("assets/fonts/{}", font.0),
-            format!("github-pages/assets/fonts/{}", font.0),
-        )
-        .unwrap();
-    }
-
     println!("done");
 }