Merge pull request #774 from CloudCannon/feat/playground

Pagefind Playground

.cloudcannon/postbuild

@@ -1 +1 @@
-npx -y pagefind@latest --site docs/public
+npx -y pagefind@latest --site docs/public --write-playground

.github/workflows/release.yml

@@ -29,6 +29,14 @@ jobs:
             target
           key: ${{ runner.os }}-stable-min165
 
+      - name: Install Nightly Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+          default: true
+          components: rustfmt, clippy, rust-src
+
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
@@ -78,6 +86,10 @@ jobs:
         working-directory: ./pagefind_ui/modular
         run: npm i && npm run build
 
+      - name: Build Playground
+        working-directory: ./pagefind_playground
+        run: npm i && npm run build
+
       - name: Build
         run: cargo build --release --verbose
       - name: Publish

.github/workflows/test.yml

@@ -23,17 +23,14 @@ jobs:
         include:
           - build: linux
             os: ubuntu-latest
-            rust: beta
             target: x86_64-unknown-linux-musl
             cross: false
           - build: macos
             os: macos-latest
-            rust: beta
             target: x86_64-apple-darwin
             cross: false
           - build: windows
             os: windows-latest
-            rust: beta
             target: x86_64-pc-windows-msvc
             cross: false
     steps:
@@ -50,20 +47,20 @@ jobs:
             ~/.cargo/git
             ~/.rustup
             target
-          key: ${{ runner.os }}-${{ matrix.rust }}-min165
+          key: ${{ runner.os }}-min165
 
       - uses: actions/setup-node@v4
         with:
           node-version: 20
 
-      - name: Install Rust
+      - name: Install Nightly Rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: ${{ matrix.rust }}
+          toolchain: nightly
           target: ${{ matrix.target }}
           override: true
           default: true
-          components: rustfmt, clippy
+          components: rustfmt, clippy, rust-src
 
       - name: Set up python 3.12
         uses: actions/setup-python@v5
@@ -82,14 +79,28 @@ jobs:
         with:
           version: ${{env.WASM_PACK_VERSION}}
 
-      - name: Build Coupled JS
-        working-directory: ./pagefind_web_js
-        run: npm i && npm run build-coupled
-
       - name: Build WASM
         working-directory: ./pagefind_web
         run: ./local_build.sh
 
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: beta
+          target: ${{ matrix.target }}
+          override: true
+          default: true
+          components: rustfmt, clippy
+
+      - name: Check versions
+        run: |
+          cargo --version
+          rustc --version
+
+      - name: Build Coupled JS
+        working-directory: ./pagefind_web_js
+        run: npm i && npm run build-coupled
+
       - name: Build UI
         working-directory: ./pagefind_ui/default
         run: npm i && npm run build
@@ -98,6 +109,10 @@ jobs:
         working-directory: ./pagefind_ui/modular
         run: npm i && npm run build
 
+      - name: Build Playground
+        working-directory: ./pagefind_playground
+        run: npm i && npm run build
+
       - name: Test Web
         working-directory: ./pagefind_web
         run: cargo test --release

.gitignore

@@ -13,6 +13,11 @@ pagefind_ui/modular/_dev_files/_pagefind/pagefind.js
 pagefind_ui/modular/_dev_files/pagefind/modular.*
 pagefind_ui/modular/_dev_files/pagefind/pagefind.js
 
+pagefind_playground/output/*
+!pagefind_playground/output/playground/
+pagefind_playground/output/playground/pagefind-playground.css
+pagefind_playground/output/playground/pagefind-playground.js
+
 npm_dist
 
 # Generated by Cargo

CHANGELOG.md

@@ -9,6 +9,9 @@
 
 ## Unreleased
 
+* Added the Pagefind Playground
+* Reduced filesizes for the Pagefind WebAssembly
+
 ## v1.3.0 (December 18, 2024)
 
 ### Core Features & Improvements

docs/content/docs/config-options.md

@@ -95,6 +95,15 @@ Keeps `index.html` at the end of search result paths. By default, a file at `ani
 |--------------------|------------------|------------------|
 | `--keep-index-url` | `KEEP_INDEX_URL` | `keep_index_url` |
 
+### Write Playground
+Writes the Pagefind playground files to `/playground` within your bundle directory. For most sites, this will make the Pagefind playground available at `/pagefind/playground/`.
+
+This defaults to false, so playground files are not written to your live site. Playground files are always available when running Pagefind with `--serve`.
+
+| CLI Flag             | ENV Variable       | Config Key         |
+|----------------------|--------------------|--------------------|
+| `--write-playground` | `WRITE_PLAYGROUND` | `write_playground` |
+
 ### Verbose
 Prints extra logging while indexing the site. Only affects the CLI, does not impact web-facing search.
 

docs/content/docs/node-api.md

@@ -45,7 +45,7 @@ All interactions with Pagefind are asynchronous, as they communicate with the na
 
 ## pagefind.createIndex
 
-Creates a Pagefind index that files can be added to.  
+Creates a Pagefind index that files can be added to.
 The index object returned is unique, and multiple calls to `pagefind.createIndex()` can be made without conflicts.
 
 ```js
@@ -64,6 +64,7 @@ const { index } = await pagefind.createIndex({
     excludeSelectors: [".my-code-blocks"],
     forceLanguage: "en",
     keepIndexUrl: false,
+    writePlayground: false,
     verbose: false,
     logfile: "debug.log"
 });
@@ -73,7 +74,7 @@ See the relevant documentation for these configuration options in the [Configuri
 
 ## index.addDirectory
 
-Indexes a directory from disk using the standard Pagefind indexing behaviour.  
+Indexes a directory from disk using the standard Pagefind indexing behaviour.
 This is equivalent to running the Pagefind binary with `--site <dir>`.
 
 ```js
@@ -91,7 +92,7 @@ const { errors, page_count } = await index.addDirectory({
 
 If the `path` provided is relative, it will be relative to the current working directory of your Node process.
 
-Optionally, a custom `glob` can be supplied which controls which files Pagefind will consume within the directory. The default is shown, and the `glob` option can be omitted entirely.  
+Optionally, a custom `glob` can be supplied which controls which files Pagefind will consume within the directory. The default is shown, and the `glob` option can be omitted entirely.
 See [Wax patterns documentation](https://github.com/olson-sean-k/wax#patterns) for more details.
 
 A response with an `errors` array containing error messages indicates that Pagefind failed to process this directory.
@@ -148,14 +149,14 @@ const { errors, file } = await index.addCustomRecord({
 
 The `url`, `content`, and `language` fields are all required. `language` should be an [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes).
 
-`meta` is optional, and is strictly a flat object of keys to string values.  
+`meta` is optional, and is strictly a flat object of keys to string values.
 See the [Metadata documentation](https://pagefind.app/docs/metadata/) for semantics.
 
-`filters` is optional, and is strictly a flat object of keys to arrays of string values.  
+`filters` is optional, and is strictly a flat object of keys to arrays of string values.
 See the [Filters documentation](https://pagefind.app/docs/filtering/) for semantics.
 
-`sort` is optional, and is strictly a flat object of keys to string values.  
-See the [Sort documentation](https://pagefind.app/docs/sorts/) for semantics.  
+`sort` is optional, and is strictly a flat object of keys to string values.
+See the [Sort documentation](https://pagefind.app/docs/sorts/) for semantics.
 *When Pagefind is processing an index, number-like strings will be sorted numerically rather than alphabetically. As such, the value passed in should be `"20"` and not `20`*
 
 A response with an `errors` array containing error messages indicates that Pagefind failed to index this content.
@@ -176,7 +177,7 @@ for (const file of files) {
 
 A response with an `errors` array containing error messages indicates that Pagefind failed to action this request.
 
-If successful, `files` will be an array containing file objects. Each object contains a `content` key with the raw data as a Uint8Array `path` key, and a `path` key which is the relative URL this file should be served at within a bundle directory. 
+If successful, `files` will be an array containing file objects. Each object contains a `content` key with the raw data as a Uint8Array `path` key, and a `path` key which is the relative URL this file should be served at within a bundle directory.
 
 ## index.writeFiles
 
@@ -208,7 +209,7 @@ Not calling this method is fine — these indexes will be cleaned up when your N
 
 ## pagefind.close
 
-Closes the Pagefind service and errors out any pending tasks, stopping the linked binary altogether.  
+Closes the Pagefind service and errors out any pending tasks, stopping the linked binary altogether.
 Called on the top-level `pagefind` object.
 
 ```js

docs/content/docs/playground.md

@@ -0,0 +1,18 @@
+---
+title: "Pagefind Playground"
+nav_title: "Pagefind Playground"
+nav_section: References
+weight: 72
+---
+
+Pagefind provides a playground page that can be used to take a deeper look into how Pagefind searches your content.
+You can explore the playground for this documentation site at [/pagefind/playground/](/pagefind/playground/).
+
+Use the playground to:
+- Determine the ideal ranking parameters to configure for your content.
+- Debug any issues with how your content is being searched.
+- Explore all data returned from Pagefind for your content.
+
+The playground is always available at `/pagefind/playground/` when running Pagefind with the [serve](/docs/config-options/#serve) option, e.g. with `--serve` via the Pagefind CLI.
+
+Setting the [write playground](/docs/config-options/#write-playground) option when indexing will write the playground files to your bundle, allowing them to be hosted as part of your site.

docs/content/docs/py-api.md

@@ -133,6 +133,7 @@ config = IndexConfig(
     verbose=True,
     logfile="index.log",
     keep_index_url=True,
+    write_playground=True,
     output_path="./output",
 )
 

docs/content/docs/ranking.md

@@ -11,6 +11,8 @@ A good example is sites with a mix of long and short pages, where the long pages
 
 Ranking parameters are configured within the `ranking` option passed to Pagefind, which can optionally contain any or all of the available parameters.
 
+> Ranking can be explored for your content via the [Pagefind playground](/docs/playground/).
+
 ## Configuring ranking parameters
 
 Ranking parameters can be passed to the JavaScript API via `pagefind.options()`:

pagefind/integration_tests/node_api/node_base/pagefind-playground-via-api.toolproof.yml

@@ -0,0 +1,40 @@
+name: Node API Base Tests > Pagefind playground can be built via the API
+platforms:
+  - linux
+  - mac
+
+steps:
+  - ref: ./background.toolproof.yml
+  - step: I have a "public/index.js" file with the content {js}
+    js: |-
+      import * as pagefind from "pagefind";
+      import fs from "fs";
+      import path from "path";
+
+      const run = async () => {
+          const { index } = await pagefind.createIndex({
+              writePlayground: true
+          });
+          await index.addCustomRecord({
+              url: "/one/",
+              content: "Testing file #1",
+              language: "en"
+          });
+          await index.addHTMLFile({sourcePath: "two/index.html", content: "<html lang='en'><body><h1>Testing file #2</h1></body></html>"});
+          await index.writeFiles();
+
+          console.log("✨!");
+      }
+
+      run();
+  - macro: I run Pagefind Node as "index.js" in "public"
+  - step: stdout should contain "✨!"
+  - step: I serve the directory "public"
+  - step: In my browser, I load "/pagefind/playground/"
+  - step: In my browser, I evaluate {js}
+    js: |-
+      let heading = await toolproof.querySelector("h1");
+      toolproof.assert_eq(heading.innerText, `Pagefind Playground`);
+
+      let val = await toolproof.querySelector("details > summary");
+      toolproof.assert_eq(val.innerText, `Details`);

pagefind/integration_tests/playground/pagefind-ui-can-programmatically-filter.toolproof.yml

@@ -0,0 +1,34 @@
+name: Pagefind Playground can be output
+steps:
+  - step: I have the environment variable "PAGEFIND_SITE" set to "public"
+  - step: I have a "public/res-zero/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><h1>title title title</h1></body></html>
+  - step: I have a "public/res-one/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><h1>title res one</h1></body></html>
+  - step: I have a "public/res-two/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><h1>title res two</h1></body></html>
+  - macro: I run Pagefind with "--write-playground"
+  - step: stdout should contain "Running Pagefind"
+  - step: The file "public/pagefind/pagefind.js" should not be empty
+  - step: I serve the directory "public"
+  - step: In my browser, I load "/pagefind/playground/"
+  - step: In my browser, I evaluate {js}
+    js: |-
+      let val = await toolproof.querySelector("details > summary");
+      toolproof.assert_eq(val.innerText, `Details`);
+
+      let search = await toolproof.querySelector("input[type='search']");
+      search.value = "title";
+      search.dispatchEvent(new Event('input', { bubbles: true }));
+  - snapshot: In my browser, the result of {js}
+    js: |-
+      // Only exists when the result data loads
+      let pinButton = await toolproof.querySelector(".result > code > button");
+      let title = await toolproof.querySelector(".result > code");
+
+      return title.innerText;
+    snapshot_content: |-
+      ╎0: [0.29812253] ☆ /res-zero/ — title title title

pagefind/integration_tests/python_api/py-pagefind-playground-via-api.toolproof.yml

@@ -0,0 +1,47 @@
+name: Python API > Pagefind playground can be built via the API
+platforms:
+  - linux
+  - mac
+
+steps:
+  - ref: ./background.toolproof.yml
+  - step: I have a "public/run.py" file with the content {python}
+    python: |2-
+      import sys
+      sys.path.append('%toolproof_process_directory%/wrappers/python/src')
+
+      import asyncio
+      import json
+      import logging
+      import os
+      from pagefind.index import PagefindIndex, IndexConfig
+
+      async def main():
+        config = IndexConfig(
+          write_playground=True,
+        )
+        async with PagefindIndex(config=config) as index:
+          await index.add_custom_record(
+            url="/one/",
+            content="Testing file #1",
+            language="en",
+          )
+          await index.add_html_file(
+            source_path="two/index.html",
+            content="<html lang='en'><body><h1>Testing file #2</h1></body></html>",
+          )
+
+        print("Complete")
+      if __name__ == "__main__":
+        asyncio.run(main())
+  - macro: I run Pagefind Python as "run.py" in "public"
+  - step: stdout should contain "Complete"
+  - step: I serve the directory "public"
+  - step: In my browser, I load "/pagefind/playground/"
+  - step: In my browser, I evaluate {js}
+    js: |-
+      let heading = await toolproof.querySelector("h1");
+      toolproof.assert_eq(heading.innerText, `Pagefind Playground`);
+
+      let val = await toolproof.querySelector("details > summary");
+      toolproof.assert_eq(val.innerText, `Details`);