From 494ffc16e264c6b8bced8da67c91cb2cb42f4737 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Thu, 7 Jul 2022 12:21:11 +0200 Subject: [PATCH 01/13] feat(@contentlayer): begin threaded building of source files EXPERIMENTAL: Consider this commit a chain of experimental changes, which likely break the pacakge. --- .../@contentlayer/source-files/package.json | 1 + .../src/fetchData/fetchAllDocuments.ts | 10 +- .../makeCacheItemFromFilePath.worker.ts | 68 + yarn.lock | 1189 +++++++++++++++-- 4 files changed, 1179 insertions(+), 89 deletions(-) create mode 100644 packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts diff --git a/packages/@contentlayer/source-files/package.json b/packages/@contentlayer/source-files/package.json index 1ff6c8a2..8fed25e9 100644 --- a/packages/@contentlayer/source-files/package.json +++ b/packages/@contentlayer/source-files/package.json @@ -43,6 +43,7 @@ "fast-glob": "^3.2.11", "gray-matter": "^4.0.3", "micromatch": "^4.0.5", + "piscina": "^3.2.0", "ts-pattern": "^4.0.2", "unified": "^10.1.2", "yaml": "^1.10.2" diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index 515322f3..db973947 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -13,6 +13,7 @@ import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' import { DocumentTypeMapState, provideDocumentTypeMapState } from './DocumentTypeMap.js' import { makeCacheItemFromFilePath } from './makeCacheItemFromFilePath.js' +import {createPool} from './makeCacheItemFromFilePath.worker.js' export const fetchAllDocuments = ({ coreSchemaDef, @@ -45,11 +46,16 @@ export const fetchAllDocuments = ({ const concurrencyLimit = os.cpus().length + const run = createPool(); + // TODO: deconstruct environment state + const environmentSeed = {}; + const { dataErrors, documents } = yield* $( pipe( allRelativeFilePaths, + // TODO: parallalize T.forEachParN(concurrencyLimit, (relativeFilePath) => - makeCacheItemFromFilePath({ + run({environmentSeed, input: { relativeFilePath, filePathPatternMap, coreSchemaDef, @@ -57,7 +63,7 @@ export const fetchAllDocuments = ({ options, previousCache, contentTypeMap, - }), + }}), ), T.map(Chunk.partitionThese), T.map(({ tuple: [errors, docs] }) => ({ dataErrors: Chunk.toArray(errors), documents: Chunk.toArray(docs) })), diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts new file mode 100644 index 00000000..e1dcec0b --- /dev/null +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -0,0 +1,68 @@ +import type * as core from '@contentlayer/core' +import type { PosixFilePath } from '@contentlayer/utils' +import type { HasConsole, OT , These } from '@contentlayer/utils/effect'; +import {pipe, T } from '@contentlayer/utils/effect' +import Pool from 'piscina'; + +import type { FetchDataError } from '../errors/index.js' +import type { ContentTypeMap, FilePathPatternMap } from '../types.js' +import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' +import * as _ from './makeCacheItemFromFilePath.js' + +export type Input = { + relativeFilePath: PosixFilePath + filePathPatternMap: FilePathPatternMap + coreSchemaDef: core.SchemaDef + contentDirPath: PosixFilePath + options: core.PluginOptions + previousCache: core.DataCache.Cache | undefined + contentTypeMap: ContentTypeMap +} + +export type DTO = { + input: Input; + environmentSeed: any; +} + +// This runs on the host, what is passed into the worker at `pool.run` has to +// be serializable. +export function createPool() { + // I believe, by default, #workers = #cpu cores, which is probably what we want? + const pool = new Pool({ + // FIXME: get path dynamically + filename: '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', + }); + + return (dto: DTO): T.Effect< + OT.HasTracer & HasConsole & HasDocumentTypeMapState, + never, + These.These +> => + pipe( + T.promise(() => pool.run(dto, {name: 'makeCacheItemFromFilePath'})), + T.chain(({_tag, value}) => + T.if_( + _tag === 'right', + () => T.succeed(value), + // FIXME: Signature claims it doesn't fail. + () => T.die(value), + ), + ), + ) +} + +// This runs in the worker, with the input coming via the "wire" from the host, +// the return value has to be serializable. +export const makeCacheItemFromFilePath = (dto: DTO) => { + // TODO: construct env + const env: OT.HasTracer & HasConsole & HasDocumentTypeMapState = undefined; + return pipe( + _.makeCacheItemFromFilePath(dto.input), + T.fold( + value => ({_tag: 'left', value} as const), + value => ({_tag: 'right', value} as const), + ), + T.provideAll(env), + T.runPromise, + ); +} diff --git a/yarn.lock b/yarn.lock index d680ebe3..2c694ebe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5,6 +5,13 @@ __metadata: version: 5 cacheKey: 8 +"@assemblyscript/loader@npm:^0.10.1": + version: 0.10.1 + resolution: "@assemblyscript/loader@npm:0.10.1" + checksum: fd1f57bdf2c55252a48c2d93fbec3c5a9ef4ca40e581e8709dd8ee437613eb47af74c8cdba011a324077eda9605d6f24983f429c2ce18b0b582ddcc5acf75c26 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.16.0": version: 7.16.0 resolution: "@babel/code-frame@npm:7.16.0" @@ -1298,6 +1305,16 @@ __metadata: languageName: node linkType: hard +"@babel/runtime-corejs3@npm:^7.10.2": + version: 7.18.6 + resolution: "@babel/runtime-corejs3@npm:7.18.6" + dependencies: + core-js-pure: ^3.20.2 + regenerator-runtime: ^0.13.4 + checksum: 55a5315b2e2541aa0dcb6193b72f8f339045d1121ff08ca87b48cbcb89447bc4550a4658e8f149c05305edd75704176ba388d780f7f0461b1b8d956a00fcf123 + languageName: node + linkType: hard + "@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.2, @babel/runtime@npm:^7.10.4, @babel/runtime@npm:^7.12.13, @babel/runtime@npm:^7.14.0, @babel/runtime@npm:^7.2.0, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.8.4": version: 7.16.0 resolution: "@babel/runtime@npm:7.16.0" @@ -1316,6 +1333,15 @@ __metadata: languageName: node linkType: hard +"@babel/runtime@npm:^7.18.3": + version: 7.18.6 + resolution: "@babel/runtime@npm:7.18.6" + dependencies: + regenerator-runtime: ^0.13.4 + checksum: 8b707b64ae0524db617d0c49933b258b96376a38307dc0be8fb42db5697608bcc1eba459acce541e376cff5ed5c5287d24db5780bd776b7c75ba2c2e26ff8a2c + languageName: node + linkType: hard + "@babel/template@npm:^7.16.0": version: 7.16.0 resolution: "@babel/template@npm:7.16.0" @@ -1658,6 +1684,7 @@ __metadata: fast-glob: ^3.2.11 gray-matter: ^4.0.3 micromatch: ^4.0.5 + piscina: ^3.2.0 ts-pattern: ^4.0.2 unified: ^10.1.2 vite: ^2.9.8 @@ -1956,6 +1983,31 @@ __metadata: languageName: node linkType: hard +"@leafac/html@npm:^1.2.1": + version: 1.3.0 + resolution: "@leafac/html@npm:1.3.0" + dependencies: + he: ^1.2.0 + sanitize-xml-string: ^1.1.0 + checksum: 5a267f485a84670a8b8608bb85401a8fb1e5b7d195d749f9a5ca531517874b4f9a53017d33f7db70010bc4a9cbf44f853b3e0cb8b8575ea4bd02c2a2b780ed48 + languageName: node + linkType: hard + +"@leafac/rehype-shiki@npm:^1.3.1": + version: 1.3.1 + resolution: "@leafac/rehype-shiki@npm:1.3.1" + dependencies: + "@leafac/html": ^1.2.1 + hast-util-to-text: ^2.0.1 + rehype-parse: ^7.0.1 + unified: ^9.2.0 + unist-util-modify-children: ^2.0.0 + peerDependencies: + shiki: ^0.9.0 + checksum: e6d16819db9e5a7b5291fa74f4aa546174f416b8f48938f79ff623e1da49f8ae78c7c093be91ff9d9a7f0aa50f0c325e2f8832afbaac4b2ec10402ec773070b8 + languageName: node + linkType: hard + "@manypkg/find-root@npm:^1.1.0": version: 1.1.0 resolution: "@manypkg/find-root@npm:1.1.0" @@ -2027,6 +2079,15 @@ __metadata: languageName: node linkType: hard +"@next/eslint-plugin-next@npm:11.1.4": + version: 11.1.4 + resolution: "@next/eslint-plugin-next@npm:11.1.4" + dependencies: + glob: 7.1.7 + checksum: f97e1a8234df3064f1aa449b393bb2699fcfc4ddeed8574727e33101100110cd25f14962495bcf69e794491f0ec36fdfab99ff1f93ae8d4d427a071a0b72df11 + languageName: node + linkType: hard + "@next/swc-android-arm-eabi@npm:12.1.6": version: 12.1.6 resolution: "@next/swc-android-arm-eabi@npm:12.1.6" @@ -2600,6 +2661,13 @@ __metadata: languageName: node linkType: hard +"@rushstack/eslint-patch@npm:^1.0.6": + version: 1.1.4 + resolution: "@rushstack/eslint-patch@npm:1.1.4" + checksum: 597bc84e2f76c7f5f2bcedd4c4b1dd5d02524167a0f67ac588e8fbbd94666297aaf0e6a53ec46afb95554164fc1169ff782841003280e4bc98e80ab6559412c6 + languageName: node + linkType: hard + "@samverschueren/stream-to-observable@npm:^0.3.0": version: 0.3.1 resolution: "@samverschueren/stream-to-observable@npm:0.3.1" @@ -3540,6 +3608,24 @@ __metadata: languageName: node linkType: hard +"@stefanprobst/rehype-shiki@npm:^2.0.4": + version: 2.2.0 + resolution: "@stefanprobst/rehype-shiki@npm:2.2.0" + dependencies: + hast-util-to-string: ^2.0.0 + json5: ^2.2.0 + parse-numeric-range: ^1.3.0 + remark-parse: ^10.0.1 + remark-rehype: ^10.1.0 + shiki-renderer-hast: ^1.1.5 + unified: ^10.1.0 + unist-util-visit: ^4.0.0 + peerDependencies: + shiki: ^0.10.0 + checksum: 036a9b360596ae35f52a89f9d9979fda1d4cccab33cd73db30b359603463054ad6d26a88b380ac81d6a43cd989d635a85a691a3998f40146058b699435228bf5 + languageName: node + linkType: hard + "@tootallnate/once@npm:1": version: 1.1.2 resolution: "@tootallnate/once@npm:1.1.2" @@ -3830,6 +3916,13 @@ __metadata: languageName: node linkType: hard +"@types/parse5@npm:^5.0.0": + version: 5.0.3 + resolution: "@types/parse5@npm:5.0.3" + checksum: d6b7495cb1850f9f2e9c5e103ede9f2d30a5320669707b105c403868adc9e4bf8d3a7ff314cc23f67826bbbbbc0e6147346ce9062ab429f099dba7a01f463919 + languageName: node + linkType: hard + "@types/prettier@npm:^2.6.0": version: 2.6.0 resolution: "@types/prettier@npm:2.6.0" @@ -3851,6 +3944,15 @@ __metadata: languageName: node linkType: hard +"@types/react-dom@npm:^17.0.9": + version: 17.0.17 + resolution: "@types/react-dom@npm:17.0.17" + dependencies: + "@types/react": ^17 + checksum: 23caf98aa03e968811560f92a2c8f451694253ebe16b670929b24eaf0e7fa62ba549abe9db0ac028a9d8a9086acd6ab9c6c773f163fa21224845edbc00ba6232 + languageName: node + linkType: hard + "@types/react-dom@npm:^18.0.3": version: 18.0.3 resolution: "@types/react-dom@npm:18.0.3" @@ -3891,6 +3993,17 @@ __metadata: languageName: node linkType: hard +"@types/react@npm:^17": + version: 17.0.47 + resolution: "@types/react@npm:17.0.47" + dependencies: + "@types/prop-types": "*" + "@types/scheduler": "*" + csstype: ^3.0.2 + checksum: 2e7fe0eb630cb77da03b6da308c58728c01b38e878118e9ff5cd8045181c8d4f32dc936e328f46a62cadb56e1fe4c5a911b5113584f93a99e1f35df7f059246b + languageName: node + linkType: hard + "@types/resolve@npm:^1.17.1": version: 1.20.1 resolution: "@types/resolve@npm:1.20.1" @@ -3921,7 +4034,7 @@ __metadata: languageName: node linkType: hard -"@types/unist@npm:*, @types/unist@npm:^2.0.0": +"@types/unist@npm:*, @types/unist@npm:^2.0.0, @types/unist@npm:^2.0.2": version: 2.0.6 resolution: "@types/unist@npm:2.0.6" checksum: 25cb860ff10dde48b54622d58b23e66214211a61c84c0f15f88d38b61aa1b53d4d46e42b557924a93178c501c166aa37e28d7f6d994aba13d24685326272d5db @@ -3974,6 +4087,23 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/parser@npm:^4.20.0": + version: 4.33.0 + resolution: "@typescript-eslint/parser@npm:4.33.0" + dependencies: + "@typescript-eslint/scope-manager": 4.33.0 + "@typescript-eslint/types": 4.33.0 + "@typescript-eslint/typescript-estree": 4.33.0 + debug: ^4.3.1 + peerDependencies: + eslint: ^5.0.0 || ^6.0.0 || ^7.0.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 102457eae1acd516211098fea081c8a2ed728522bbda7f5a557b6ef23d88970514f9a0f6285d53fca134d3d4d7d17822b5d5e12438d5918df4d1f89cc9e67d57 + languageName: node + linkType: hard + "@typescript-eslint/parser@npm:^5.23.0": version: 5.23.0 resolution: "@typescript-eslint/parser@npm:5.23.0" @@ -3991,6 +4121,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/scope-manager@npm:4.33.0": + version: 4.33.0 + resolution: "@typescript-eslint/scope-manager@npm:4.33.0" + dependencies: + "@typescript-eslint/types": 4.33.0 + "@typescript-eslint/visitor-keys": 4.33.0 + checksum: 9a25fb7ba7c725ea7227a24d315b0f6aacbad002e2549a049edf723c1d3615c22f5c301f0d7d615b377f2cdf2f3519d97e79af0c459de6ef8d2aaf0906dff13e + languageName: node + linkType: hard + "@typescript-eslint/scope-manager@npm:5.23.0": version: 5.23.0 resolution: "@typescript-eslint/scope-manager@npm:5.23.0" @@ -4017,6 +4157,13 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/types@npm:4.33.0": + version: 4.33.0 + resolution: "@typescript-eslint/types@npm:4.33.0" + checksum: 3baae1ca35872421b4eb60f5d3f3f32dc1d513f2ae0a67dee28c7d159fd7a43ed0d11a8a5a0f0c2d38507ffa036fc7c511cb0f18a5e8ac524b3ebde77390ec53 + languageName: node + linkType: hard + "@typescript-eslint/types@npm:5.23.0": version: 5.23.0 resolution: "@typescript-eslint/types@npm:5.23.0" @@ -4024,6 +4171,24 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/typescript-estree@npm:4.33.0": + version: 4.33.0 + resolution: "@typescript-eslint/typescript-estree@npm:4.33.0" + dependencies: + "@typescript-eslint/types": 4.33.0 + "@typescript-eslint/visitor-keys": 4.33.0 + debug: ^4.3.1 + globby: ^11.0.3 + is-glob: ^4.0.1 + semver: ^7.3.5 + tsutils: ^3.21.0 + peerDependenciesMeta: + typescript: + optional: true + checksum: 2566984390c76bd95f43240057215c068c69769e406e27aba41e9f21fd300074d6772e4983fa58fe61e80eb5550af1548d2e31e80550d92ba1d051bb00fe6f5c + languageName: node + linkType: hard + "@typescript-eslint/typescript-estree@npm:5.23.0": version: 5.23.0 resolution: "@typescript-eslint/typescript-estree@npm:5.23.0" @@ -4058,6 +4223,16 @@ __metadata: languageName: node linkType: hard +"@typescript-eslint/visitor-keys@npm:4.33.0": + version: 4.33.0 + resolution: "@typescript-eslint/visitor-keys@npm:4.33.0" + dependencies: + "@typescript-eslint/types": 4.33.0 + eslint-visitor-keys: ^2.0.0 + checksum: 59953e474ad4610c1aa23b2b1a964445e2c6201521da6367752f37939d854352bbfced5c04ea539274065e012b1337ba3ffa49c2647a240a4e87155378ba9873 + languageName: node + linkType: hard + "@typescript-eslint/visitor-keys@npm:5.23.0": version: 5.23.0 resolution: "@typescript-eslint/visitor-keys@npm:5.23.0" @@ -4120,7 +4295,7 @@ __metadata: languageName: node linkType: hard -"acorn-node@npm:^1.6.1": +"acorn-node@npm:^1.8.2": version: 1.8.2 resolution: "acorn-node@npm:1.8.2" dependencies: @@ -4472,10 +4647,10 @@ __metadata: languageName: node linkType: hard -"arg@npm:^5.0.1": - version: 5.0.1 - resolution: "arg@npm:5.0.1" - checksum: 9aefbcb1204f8dbd541a045bfe99b6515b4dc697c2f704ef2bb5e9fe5464575d97571e91e673a6f23ad72dd1cc24d7d8cf2d1d828e72c08e4d4f6f9237adc761 +"arg@npm:^5.0.2": + version: 5.0.2 + resolution: "arg@npm:5.0.2" + checksum: 6c69ada1a9943d332d9e5382393e897c500908d91d5cb735a01120d5f71daf1b339b7b8980cbeaba8fd1afc68e658a739746179e4315a26e8a28951ff9930078 languageName: node linkType: hard @@ -4495,6 +4670,16 @@ __metadata: languageName: node linkType: hard +"aria-query@npm:^4.2.2": + version: 4.2.2 + resolution: "aria-query@npm:4.2.2" + dependencies: + "@babel/runtime": ^7.10.2 + "@babel/runtime-corejs3": ^7.10.2 + checksum: 38401a9a400f26f3dcc24b84997461a16b32869a9893d323602bed8da40a8bcc0243b8d2880e942249a1496cea7a7de769e93d21c0baa439f01e1ee936fed665 + languageName: node + linkType: hard + "arr-diff@npm:^4.0.0": version: 4.0.0 resolution: "arr-diff@npm:4.0.0" @@ -4550,6 +4735,26 @@ __metadata: languageName: node linkType: hard +"array-includes@npm:^3.1.5": + version: 3.1.5 + resolution: "array-includes@npm:3.1.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.4 + es-abstract: ^1.19.5 + get-intrinsic: ^1.1.1 + is-string: ^1.0.7 + checksum: f6f24d834179604656b7bec3e047251d5cc87e9e87fab7c175c61af48e80e75acd296017abcde21fb52292ab6a2a449ab2ee37213ee48c8709f004d75983f9c5 + languageName: node + linkType: hard + +"array-iterate@npm:^1.0.0": + version: 1.1.4 + resolution: "array-iterate@npm:1.1.4" + checksum: 8adc65525dfa871577b7ab91b41efd61d29c4067a08ec927340d6975e45797b9f04254dda115e366fbef11fb49277ac1c166405389886c7a251e1eddca89bd08 + languageName: node + linkType: hard + "array-timsort@npm:^1.0.3": version: 1.0.3 resolution: "array-timsort@npm:1.0.3" @@ -4582,6 +4787,18 @@ __metadata: languageName: node linkType: hard +"array.prototype.flatmap@npm:^1.3.0": + version: 1.3.0 + resolution: "array.prototype.flatmap@npm:1.3.0" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + es-abstract: ^1.19.2 + es-shim-unscopables: ^1.0.0 + checksum: 818538f39409c4045d874be85df0dbd195e1446b14d22f95bdcfefea44ae77db44e42dcd89a559254ec5a7c8b338cfc986cc6d641e3472f9a5326b21eb2976a2 + languageName: node + linkType: hard + "arrify@npm:^1.0.1": version: 1.0.1 resolution: "arrify@npm:1.0.1" @@ -4648,6 +4865,13 @@ __metadata: languageName: node linkType: hard +"ast-types-flow@npm:^0.0.7": + version: 0.0.7 + resolution: "ast-types-flow@npm:0.0.7" + checksum: a26dcc2182ffee111cad7c471759b0bda22d3b7ebacf27c348b22c55f16896b18ab0a4d03b85b4020dce7f3e634b8f00b593888f622915096ea1927fa51866c4 + languageName: node + linkType: hard + "astring@npm:^1.6.0": version: 1.7.5 resolution: "astring@npm:1.7.5" @@ -4774,6 +4998,13 @@ __metadata: languageName: node linkType: hard +"axe-core@npm:^4.4.2": + version: 4.4.2 + resolution: "axe-core@npm:4.4.2" + checksum: 93fbb36c5ac8ab5e67e49678a6f7be0dc799a9f560edd95cca1f0a8183def8c50205972366b9941a3ea2b20224a1fe230e6d87ef38cb6db70472ed1b694febd1 + languageName: node + linkType: hard + "axios@npm:^0.21.0, axios@npm:^0.21.4": version: 0.21.4 resolution: "axios@npm:0.21.4" @@ -4783,6 +5014,13 @@ __metadata: languageName: node linkType: hard +"axobject-query@npm:^2.2.0": + version: 2.2.0 + resolution: "axobject-query@npm:2.2.0" + checksum: 96b8c7d807ca525f41ad9b286186e2089b561ba63a6d36c3e7d73dc08150714660995c7ad19cda05784458446a0793b45246db45894631e13853f48c1aa3117f + languageName: node + linkType: hard + "babel-code-frame@npm:^6.26.0": version: 6.26.0 resolution: "babel-code-frame@npm:6.26.0" @@ -4906,7 +5144,7 @@ __metadata: languageName: node linkType: hard -"base64-js@npm:^1.0.2, base64-js@npm:^1.3.1": +"base64-js@npm:^1.0.2, base64-js@npm:^1.2.0, base64-js@npm:^1.3.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" checksum: 669632eb3745404c2f822a18fc3a0122d2f9a7a13f7fb8b5823ee19d1d2ff9ee5b52c53367176ea4ad093c332fd5ab4bd0ebae5a8e27917a4105a4cfc86b1005 @@ -5287,17 +5525,16 @@ __metadata: linkType: hard "browserslist@npm:^4.20.3": - version: 4.20.3 - resolution: "browserslist@npm:4.20.3" + version: 4.21.1 + resolution: "browserslist@npm:4.21.1" dependencies: - caniuse-lite: ^1.0.30001332 - electron-to-chromium: ^1.4.118 - escalade: ^3.1.1 - node-releases: ^2.0.3 - picocolors: ^1.0.0 + caniuse-lite: ^1.0.30001359 + electron-to-chromium: ^1.4.172 + node-releases: ^2.0.5 + update-browserslist-db: ^1.0.4 bin: browserslist: cli.js - checksum: 1e4b719ac2ca0fe235218a606e8b8ef16b8809e0973b924158c39fbc435a0b0fe43437ea52dd6ef5ad2efcb83fcb07431244e472270177814217f7c563651f7d + checksum: 4904a9ded0702381adc495e003e7f77970abb7f8c8b8edd9e54f026354b5a96b1bddc26e6d9a7df9f043e468ecd2fcff2c8f40fc489909a042880117c2aca8ff languageName: node linkType: hard @@ -5569,13 +5806,20 @@ __metadata: languageName: node linkType: hard -"caniuse-lite@npm:^1.0.30001332, caniuse-lite@npm:^1.0.30001335": +"caniuse-lite@npm:^1.0.30001332": version: 1.0.30001339 resolution: "caniuse-lite@npm:1.0.30001339" checksum: c974676c6e38692ab5a274c460557476f1d167166493249059b5595dc3166942a30bb030dd4a6f6dcbc28fb53c741b5c95fc0f82b043def296e6a49b32b68478 languageName: node linkType: hard +"caniuse-lite@npm:^1.0.30001335, caniuse-lite@npm:^1.0.30001359": + version: 1.0.30001363 + resolution: "caniuse-lite@npm:1.0.30001363" + checksum: 8dfcb2fa97724349cbbe61d988810bd90bfb40106a289ed6613188fa96dd1f5885c7e9924e46bb30a641bd1579ec34096fdc2b21b47d8500f8a2bfb0db069323 + languageName: node + linkType: hard + "capture-stack-trace@npm:^1.0.0": version: 1.0.1 resolution: "capture-stack-trace@npm:1.0.1" @@ -6453,6 +6697,13 @@ __metadata: languageName: node linkType: hard +"core-js-pure@npm:^3.20.2": + version: 3.23.3 + resolution: "core-js-pure@npm:3.23.3" + checksum: 09a477a56963ca4409ca383d36429ea3b51b658ff85e94331a510543c77c4d1b44cb6b305b0f185d729eb059c71f1289c62fdec6371ff46ce838a16988cdcb2e + languageName: node + linkType: hard + "core-js@npm:^2.4.0, core-js@npm:^2.5.0": version: 2.6.12 resolution: "core-js@npm:2.6.12" @@ -6934,6 +7185,13 @@ __metadata: languageName: node linkType: hard +"damerau-levenshtein@npm:^1.0.8": + version: 1.0.8 + resolution: "damerau-levenshtein@npm:1.0.8" + checksum: d240b7757544460ae0586a341a53110ab0a61126570ef2d8c731e3eab3f0cb6e488e2609e6a69b46727635de49be20b071688698744417ff1b6c1d7ccd03e0de + languageName: node + linkType: hard + "dashdash@npm:^1.12.0": version: 1.14.1 resolution: "dashdash@npm:1.14.1" @@ -7051,7 +7309,7 @@ __metadata: languageName: node linkType: hard -"debug@npm:^4.3.2": +"debug@npm:^4.3.2, debug@npm:^4.3.4": version: 4.3.4 resolution: "debug@npm:4.3.4" dependencies: @@ -7166,6 +7424,16 @@ __metadata: languageName: node linkType: hard +"define-properties@npm:^1.1.4": + version: 1.1.4 + resolution: "define-properties@npm:1.1.4" + dependencies: + has-property-descriptors: ^1.0.0 + object-keys: ^1.1.1 + checksum: ce0aef3f9eb193562b5cfb79b2d2c86b6a109dfc9fdcb5f45d680631a1a908c06824ddcdb72b7573b54e26ace07f0a23420aaba0d5c627b34d2c1de8ef527e2b + languageName: node + linkType: hard + "define-property@npm:^0.2.5": version: 0.2.5 resolution: "define-property@npm:0.2.5" @@ -7267,16 +7535,16 @@ __metadata: languageName: node linkType: hard -"detective@npm:^5.2.0": - version: 5.2.0 - resolution: "detective@npm:5.2.0" +"detective@npm:^5.2.1": + version: 5.2.1 + resolution: "detective@npm:5.2.1" dependencies: - acorn-node: ^1.6.1 + acorn-node: ^1.8.2 defined: ^1.0.0 - minimist: ^1.1.1 + minimist: ^1.2.6 bin: detective: bin/detective.js - checksum: 2ab266aecbd695b42e4703cfa560178ceac4308a74baece58185775426e65573d563d84f33e6a3b28ef3a544aa0c039c0730ada939c6458862e6643f66044f32 + checksum: dc4601bbc6be850edb3c2dab7a0eaf5a6169a15ad201679c66d40ea1986df816eeaecd590047f15b0780285f3eeea13b82dca0d4c52a47e744a571e326a72dc9 languageName: node linkType: hard @@ -7541,10 +7809,10 @@ __metadata: languageName: node linkType: hard -"electron-to-chromium@npm:^1.4.118": - version: 1.4.137 - resolution: "electron-to-chromium@npm:1.4.137" - checksum: 639d7b94906efafcf363519c3698eecc44be46755a6a5cdc9088954329978866cc93fbd57e08b97290599b68d5226243d21de9fa50be416b8a5d3fa8fd42c3a0 +"electron-to-chromium@npm:^1.4.172": + version: 1.4.177 + resolution: "electron-to-chromium@npm:1.4.177" + checksum: e373df9b001c9a77a33b78ab4b8dbe6ee4175eb630c7d8dbe3671eb50be62a91c220cec71d56c2da3c532679ee692fe34715b915b900dec0962c08a983d06a86 languageName: node linkType: hard @@ -7586,6 +7854,13 @@ __metadata: languageName: node linkType: hard +"emoji-regex@npm:^9.2.2": + version: 9.2.2 + resolution: "emoji-regex@npm:9.2.2" + checksum: 8487182da74aabd810ac6d6f1994111dfc0e331b01271ae01ec1eb0ad7b5ecc2bbbbd2f053c05cb55a1ac30449527d819bbfbf0e3de1023db308cbcb47f86601 + languageName: node + linkType: hard + "emojis-list@npm:^2.0.0": version: 2.1.0 resolution: "emojis-list@npm:2.1.0" @@ -7729,6 +8004,46 @@ __metadata: languageName: node linkType: hard +"es-abstract@npm:^1.19.2, es-abstract@npm:^1.19.5": + version: 1.20.1 + resolution: "es-abstract@npm:1.20.1" + dependencies: + call-bind: ^1.0.2 + es-to-primitive: ^1.2.1 + function-bind: ^1.1.1 + function.prototype.name: ^1.1.5 + get-intrinsic: ^1.1.1 + get-symbol-description: ^1.0.0 + has: ^1.0.3 + has-property-descriptors: ^1.0.0 + has-symbols: ^1.0.3 + internal-slot: ^1.0.3 + is-callable: ^1.2.4 + is-negative-zero: ^2.0.2 + is-regex: ^1.1.4 + is-shared-array-buffer: ^1.0.2 + is-string: ^1.0.7 + is-weakref: ^1.0.2 + object-inspect: ^1.12.0 + object-keys: ^1.1.1 + object.assign: ^4.1.2 + regexp.prototype.flags: ^1.4.3 + string.prototype.trimend: ^1.0.5 + string.prototype.trimstart: ^1.0.5 + unbox-primitive: ^1.0.2 + checksum: 28da27ae0ed9c76df7ee8ef5c278df79dcfdb554415faf7068bb7c58f8ba8e2a16bfb59e586844be6429ab4c302ca7748979d48442224cb1140b051866d74b7f + languageName: node + linkType: hard + +"es-shim-unscopables@npm:^1.0.0": + version: 1.0.0 + resolution: "es-shim-unscopables@npm:1.0.0" + dependencies: + has: ^1.0.3 + checksum: 83e95cadbb6ee44d3644dfad60dcad7929edbc42c85e66c3e99aefd68a3a5c5665f2686885cddb47dfeabfd77bd5ea5a7060f2092a955a729bbd8834f0d86fa1 + languageName: node + linkType: hard + "es-to-primitive@npm:^1.2.1": version: 1.2.1 resolution: "es-to-primitive@npm:1.2.1" @@ -8098,6 +8413,30 @@ __metadata: languageName: node linkType: hard +"eslint-config-next@npm:^11.0.1": + version: 11.1.4 + resolution: "eslint-config-next@npm:11.1.4" + dependencies: + "@next/eslint-plugin-next": 11.1.4 + "@rushstack/eslint-patch": ^1.0.6 + "@typescript-eslint/parser": ^4.20.0 + eslint-import-resolver-node: ^0.3.4 + eslint-import-resolver-typescript: ^2.4.0 + eslint-plugin-import: ^2.22.1 + eslint-plugin-jsx-a11y: ^6.4.1 + eslint-plugin-react: ^7.23.1 + eslint-plugin-react-hooks: ^4.2.0 + peerDependencies: + eslint: ^7.23.0 + next: ">=10.2.0" + typescript: ">=3.3.1" + peerDependenciesMeta: + typescript: + optional: true + checksum: 47d3b885c8045a8859178df495cece4f460b8812eeda01b43e0ad2434b64743cdd4255ecd759217f23ebaa90cb57ecbc68ba9cdda94da76411afab36f382ca75 + languageName: node + linkType: hard + "eslint-config-prettier@npm:^8.5.0": version: 8.5.0 resolution: "eslint-config-prettier@npm:8.5.0" @@ -8109,7 +8448,7 @@ __metadata: languageName: node linkType: hard -"eslint-import-resolver-node@npm:^0.3.6": +"eslint-import-resolver-node@npm:^0.3.4, eslint-import-resolver-node@npm:^0.3.6": version: 0.3.6 resolution: "eslint-import-resolver-node@npm:0.3.6" dependencies: @@ -8119,6 +8458,22 @@ __metadata: languageName: node linkType: hard +"eslint-import-resolver-typescript@npm:^2.4.0": + version: 2.7.1 + resolution: "eslint-import-resolver-typescript@npm:2.7.1" + dependencies: + debug: ^4.3.4 + glob: ^7.2.0 + is-glob: ^4.0.3 + resolve: ^1.22.0 + tsconfig-paths: ^3.14.1 + peerDependencies: + eslint: "*" + eslint-plugin-import: "*" + checksum: 1d81b657b1f73bf95b8f0b745c0305574b91630c1db340318f3ca8918e206fce20a933b95e7c419338cc4452cb80bb2b2d92acaf01b6aa315c78a332d832545c + languageName: node + linkType: hard + "eslint-module-utils@npm:^2.7.3": version: 2.7.3 resolution: "eslint-module-utils@npm:2.7.3" @@ -8129,7 +8484,7 @@ __metadata: languageName: node linkType: hard -"eslint-plugin-import@npm:^2.26.0": +"eslint-plugin-import@npm:^2.22.1, eslint-plugin-import@npm:^2.26.0": version: 2.26.0 resolution: "eslint-plugin-import@npm:2.26.0" dependencies: @@ -8152,6 +8507,38 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-jsx-a11y@npm:^6.4.1": + version: 6.6.0 + resolution: "eslint-plugin-jsx-a11y@npm:6.6.0" + dependencies: + "@babel/runtime": ^7.18.3 + aria-query: ^4.2.2 + array-includes: ^3.1.5 + ast-types-flow: ^0.0.7 + axe-core: ^4.4.2 + axobject-query: ^2.2.0 + damerau-levenshtein: ^1.0.8 + emoji-regex: ^9.2.2 + has: ^1.0.3 + jsx-ast-utils: ^3.3.1 + language-tags: ^1.0.5 + minimatch: ^3.1.2 + semver: ^6.3.0 + peerDependencies: + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + checksum: d9da9a3ec71137c12519289c63e71250d5d78d4b7729b84e7e12edf1aad993083f23303d9b62359591b2f8aababb1bbec032cd84f1425e759b11a055e3acd144 + languageName: node + linkType: hard + +"eslint-plugin-react-hooks@npm:^4.2.0": + version: 4.6.0 + resolution: "eslint-plugin-react-hooks@npm:4.6.0" + peerDependencies: + eslint: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 + checksum: 23001801f14c1d16bf0a837ca7970d9dd94e7b560384b41db378b49b6e32dc43d6e2790de1bd737a652a86f81a08d6a91f402525061b47719328f586a57e86c3 + languageName: node + linkType: hard + "eslint-plugin-react-hooks@npm:^4.5.0": version: 4.5.0 resolution: "eslint-plugin-react-hooks@npm:4.5.0" @@ -8161,6 +8548,30 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-react@npm:^7.23.1": + version: 7.30.1 + resolution: "eslint-plugin-react@npm:7.30.1" + dependencies: + array-includes: ^3.1.5 + array.prototype.flatmap: ^1.3.0 + doctrine: ^2.1.0 + estraverse: ^5.3.0 + jsx-ast-utils: ^2.4.1 || ^3.0.0 + minimatch: ^3.1.2 + object.entries: ^1.1.5 + object.fromentries: ^2.0.5 + object.hasown: ^1.1.1 + object.values: ^1.1.5 + prop-types: ^15.8.1 + resolve: ^2.0.0-next.3 + semver: ^6.3.0 + string.prototype.matchall: ^4.0.7 + peerDependencies: + eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 + checksum: 553fb9ece6beb7c14cf6f84670c786c8ac978c2918421994dcc4edd2385302022e5d5ac4a39fafdb35954e29cecddefed61758040c3c530cafcf651f674a9d51 + languageName: node + linkType: hard + "eslint-plugin-simple-import-sort@npm:^7.0.0": version: 7.0.0 resolution: "eslint-plugin-simple-import-sort@npm:7.0.0" @@ -8315,7 +8726,7 @@ __metadata: languageName: node linkType: hard -"estraverse@npm:^5.1.0, estraverse@npm:^5.2.0": +"estraverse@npm:^5.1.0, estraverse@npm:^5.2.0, estraverse@npm:^5.3.0": version: 5.3.0 resolution: "estraverse@npm:5.3.0" checksum: 072780882dc8416ad144f8fe199628d2b3e7bbc9989d9ed43795d2c90309a2047e6bc5979d7e2322a341163d22cfad9e21f4110597fe487519697389497e4e2b @@ -8406,6 +8817,13 @@ __metadata: languageName: node linkType: hard +"eventemitter-asyncresource@npm:^1.0.0": + version: 1.0.0 + resolution: "eventemitter-asyncresource@npm:1.0.0" + checksum: 3cfbbc3490bd429a165bff6336289ff810f7df214796f25000d2097a5a0883eae51542a78674916ff99bbd4c66811911b310df1cb4fc96dfc9546ba9dfc89f8f + languageName: node + linkType: hard + "events@npm:^3.0.0": version: 3.3.0 resolution: "events@npm:3.3.0" @@ -8555,6 +8973,25 @@ __metadata: languageName: unknown linkType: soft +"examples-gatsby-docs@workspace:examples/gatsbydocs": + version: 0.0.0-use.local + resolution: "examples-gatsby-docs@workspace:examples/gatsbydocs" + dependencies: + "@leafac/rehype-shiki": ^1.3.1 + "@stefanprobst/rehype-shiki": ^2.0.4 + "@types/react": 18.0.9 + "@types/react-dom": ^17.0.9 + contentlayer: "workspace:*" + eslint-config-next: ^11.0.1 + next: 12.1.6 + next-contentlayer: "workspace:*" + react: 18.1.0 + react-dom: 18.1.0 + shiki: ^0.9.4 + typescript: 4.6.4 + languageName: unknown + linkType: soft + "examples-starter-js@workspace:examples/archive/starter-js": version: 0.0.0-use.local resolution: "examples-starter-js@workspace:examples/archive/starter-js" @@ -9404,6 +9841,18 @@ fsevents@~2.3.2: languageName: node linkType: hard +"function.prototype.name@npm:^1.1.5": + version: 1.1.5 + resolution: "function.prototype.name@npm:1.1.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + es-abstract: ^1.19.0 + functions-have-names: ^1.2.2 + checksum: acd21d733a9b649c2c442f067567743214af5fa248dbeee69d8278ce7df3329ea5abac572be9f7470b4ec1cd4d8f1040e3c5caccf98ebf2bf861a0deab735c27 + languageName: node + linkType: hard + "functional-red-black-tree@npm:^1.0.1": version: 1.0.1 resolution: "functional-red-black-tree@npm:1.0.1" @@ -9411,6 +9860,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"functions-have-names@npm:^1.2.2": + version: 1.2.3 + resolution: "functions-have-names@npm:1.2.3" + checksum: c3f1f5ba20f4e962efb71344ce0a40722163e85bee2101ce25f88214e78182d2d2476aa85ef37950c579eb6cf6ee811c17b3101bb84004bb75655f3e33f3fdb5 + languageName: node + linkType: hard + "gauge@npm:~2.7.3": version: 2.7.4 resolution: "gauge@npm:2.7.4" @@ -9602,6 +10058,20 @@ fsevents@~2.3.2: languageName: node linkType: hard +"glob@npm:7.1.7": + version: 7.1.7 + resolution: "glob@npm:7.1.7" + dependencies: + fs.realpath: ^1.0.0 + inflight: ^1.0.4 + inherits: 2 + minimatch: ^3.0.4 + once: ^1.3.0 + path-is-absolute: ^1.0.0 + checksum: b61f48973bbdcf5159997b0874a2165db572b368b931135832599875919c237fc05c12984e38fe828e69aa8a921eb0e8a4997266211c517c9cfaae8a93988bb8 + languageName: node + linkType: hard + "glob@npm:^7.0.0, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.7, glob@npm:~7.2.0": version: 7.2.0 resolution: "glob@npm:7.2.0" @@ -9616,8 +10086,22 @@ fsevents@~2.3.2: languageName: node linkType: hard -"global-prefix@npm:^3.0.0": - version: 3.0.0 +"glob@npm:^7.2.0": + version: 7.2.3 + resolution: "glob@npm:7.2.3" + dependencies: + fs.realpath: ^1.0.0 + inflight: ^1.0.4 + inherits: 2 + minimatch: ^3.1.1 + once: ^1.3.0 + path-is-absolute: ^1.0.0 + checksum: 29452e97b38fa704dabb1d1045350fb2467cf0277e155aa9ff7077e90ad81d1ea9d53d3ee63bd37c05b09a065e90f16aec4a65f5b8de401d1dac40bc5605d133 + languageName: node + linkType: hard + +"global-prefix@npm:^3.0.0": + version: 3.0.0 resolution: "global-prefix@npm:3.0.0" dependencies: ini: ^1.3.5 @@ -9683,7 +10167,7 @@ fsevents@~2.3.2: languageName: node linkType: hard -"globby@npm:^11.0.4": +"globby@npm:^11.0.3, globby@npm:^11.0.4": version: 11.1.0 resolution: "globby@npm:11.1.0" dependencies: @@ -9786,6 +10270,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"has-bigints@npm:^1.0.2": + version: 1.0.2 + resolution: "has-bigints@npm:1.0.2" + checksum: 390e31e7be7e5c6fe68b81babb73dfc35d413604d7ee5f56da101417027a4b4ce6a27e46eff97ad040c835b5d228676eae99a9b5c3bc0e23c8e81a49241ff45b + languageName: node + linkType: hard + "has-flag@npm:^1.0.0": version: 1.0.0 resolution: "has-flag@npm:1.0.0" @@ -9821,6 +10312,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"has-property-descriptors@npm:^1.0.0": + version: 1.0.0 + resolution: "has-property-descriptors@npm:1.0.0" + dependencies: + get-intrinsic: ^1.1.1 + checksum: a6d3f0a266d0294d972e354782e872e2fe1b6495b321e6ef678c9b7a06a40408a6891817350c62e752adced73a94ac903c54734fee05bf65b1905ee1368194bb + languageName: node + linkType: hard + "has-symbols@npm:^1.0.1, has-symbols@npm:^1.0.2": version: 1.0.2 resolution: "has-symbols@npm:1.0.2" @@ -9828,6 +10328,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"has-symbols@npm:^1.0.3": + version: 1.0.3 + resolution: "has-symbols@npm:1.0.3" + checksum: a054c40c631c0d5741a8285010a0777ea0c068f99ed43e5d6eb12972da223f8af553a455132fdb0801bdcfa0e0f443c0c03a68d8555aa529b3144b446c3f2410 + languageName: node + linkType: hard + "has-tostringtag@npm:^1.0.0": version: 1.0.0 resolution: "has-tostringtag@npm:1.0.0" @@ -9927,6 +10434,27 @@ fsevents@~2.3.2: languageName: node linkType: hard +"hast-util-from-parse5@npm:^6.0.0": + version: 6.0.1 + resolution: "hast-util-from-parse5@npm:6.0.1" + dependencies: + "@types/parse5": ^5.0.0 + hastscript: ^6.0.0 + property-information: ^5.0.0 + vfile: ^4.0.0 + vfile-location: ^3.2.0 + web-namespaces: ^1.0.0 + checksum: 4daa78201468af7779161e7caa2513c329830778e0528481ab16b3e1bcef4b831f6285b526aacdddbee802f3bd9d64df55f80f010591ea1916da535e3a923b83 + languageName: node + linkType: hard + +"hast-util-is-element@npm:^1.0.0": + version: 1.1.0 + resolution: "hast-util-is-element@npm:1.1.0" + checksum: 30fad3f65e7ab2f0efd5db9e7344d0820b70971988dfe79f62d8447598b2a1ce8a59cd4bfc05ae0d9a1c451b9b53cbe1023743d7eac764d64720b6b73475f62f + languageName: node + linkType: hard + "hast-util-is-element@npm:^2.0.0": version: 2.1.1 resolution: "hast-util-is-element@npm:2.1.1" @@ -9944,6 +10472,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"hast-util-parse-selector@npm:^3.0.0": + version: 3.1.0 + resolution: "hast-util-parse-selector@npm:3.1.0" + dependencies: + "@types/hast": ^2.0.0 + checksum: 8be1a2334652866b40fde72a8b7d0867a791ce8a70d15fd7bb44b9a4f349913b77999e5add41900466bc9461c6b0fdea391875ef534b33cacf7a2aee9d8e447c + languageName: node + linkType: hard + "hast-util-to-estree@npm:^2.0.0": version: 2.0.2 resolution: "hast-util-to-estree@npm:2.0.2" @@ -9984,6 +10521,26 @@ fsevents@~2.3.2: languageName: node linkType: hard +"hast-util-to-string@npm:^2.0.0": + version: 2.0.0 + resolution: "hast-util-to-string@npm:2.0.0" + dependencies: + "@types/hast": ^2.0.0 + checksum: 0c087f8dee4238741cbad65d28adb8bf800252c763a3c643df2fcb4ef97232056837928c2ae73f841f310e4d336c3b183ee380a5e6eb24bda5c117f78ed600d4 + languageName: node + linkType: hard + +"hast-util-to-text@npm:^2.0.1": + version: 2.0.1 + resolution: "hast-util-to-text@npm:2.0.1" + dependencies: + hast-util-is-element: ^1.0.0 + repeat-string: ^1.0.0 + unist-util-find-after: ^3.0.0 + checksum: 4e7960b414b7a6b2f0180e4af416cd8ae3c7ba1531d7eaec7e6dc9509daf88308784bbf5b94885384dccc42abcb74cc6cc26755c76914d646f32aa6bc32ea34b + languageName: node + linkType: hard + "hast-util-to-text@npm:^3.0.0": version: 3.1.1 resolution: "hast-util-to-text@npm:3.1.1" @@ -10015,6 +10572,46 @@ fsevents@~2.3.2: languageName: node linkType: hard +"hastscript@npm:^7.0.2": + version: 7.0.2 + resolution: "hastscript@npm:7.0.2" + dependencies: + "@types/hast": ^2.0.0 + comma-separated-tokens: ^2.0.0 + hast-util-parse-selector: ^3.0.0 + property-information: ^6.0.0 + space-separated-tokens: ^2.0.0 + checksum: ee33aff714b12f9f83049550956c7fb3e5ac7bdd20e77b57dc01b66de06e8bb0b3ba24153d4b6a1d7fa660bfef91125ac29e1bb04fb628e30d11097d28037235 + languageName: node + linkType: hard + +"hdr-histogram-js@npm:^2.0.1": + version: 2.0.3 + resolution: "hdr-histogram-js@npm:2.0.3" + dependencies: + "@assemblyscript/loader": ^0.10.1 + base64-js: ^1.2.0 + pako: ^1.0.3 + checksum: 7bb252ba3596bed72b90427ffc6f6fa332a460c4810788faa9b9a743f7ac6f1cb42dccd7ae7555740f0a8c0602884944d00d1ccfb746af4976a816772361a6d6 + languageName: node + linkType: hard + +"hdr-histogram-percentiles-obj@npm:^3.0.0": + version: 3.0.0 + resolution: "hdr-histogram-percentiles-obj@npm:3.0.0" + checksum: ab238edcb38d9b60d23ca53da0ecd9a6b1c8ee9a49e30a6146bd3f8f70f26244652f28b79974157c00504e7ddf3129e0ddb217baf71d32330e3fae0105bf30ed + languageName: node + linkType: hard + +"he@npm:^1.2.0": + version: 1.2.0 + resolution: "he@npm:1.2.0" + bin: + he: bin/he + checksum: 3d4d6babccccd79c5c5a3f929a68af33360d6445587d628087f39a965079d84f18ce9c3d3f917ee1e3978916fc833bb8b29377c3b403f919426f91bc6965e7a7 + languageName: node + linkType: hard + "hex-color-regex@npm:^1.1.0": version: 1.1.0 resolution: "hex-color-regex@npm:1.1.0" @@ -10734,6 +11331,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"is-core-module@npm:^2.9.0": + version: 2.9.0 + resolution: "is-core-module@npm:2.9.0" + dependencies: + has: ^1.0.3 + checksum: b27034318b4b462f1c8f1dfb1b32baecd651d891a4e2d1922135daeff4141dfced2b82b07aef83ef54275c4a3526aa38da859223664d0868ca24182badb784ce + languageName: node + linkType: hard + "is-data-descriptor@npm:^0.1.4": version: 0.1.4 resolution: "is-data-descriptor@npm:0.1.4" @@ -10924,6 +11530,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"is-negative-zero@npm:^2.0.2": + version: 2.0.2 + resolution: "is-negative-zero@npm:2.0.2" + checksum: f3232194c47a549da60c3d509c9a09be442507616b69454716692e37ae9f37c4dea264fb208ad0c9f3efd15a796a46b79df07c7e53c6227c32170608b809149a + languageName: node + linkType: hard + "is-number-object@npm:^1.0.4": version: 1.0.6 resolution: "is-number-object@npm:1.0.6" @@ -10979,6 +11592,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"is-plain-obj@npm:^2.0.0": + version: 2.1.0 + resolution: "is-plain-obj@npm:2.1.0" + checksum: cec9100678b0a9fe0248a81743041ed990c2d4c99f893d935545cfbc42876cbe86d207f3b895700c690ad2fa520e568c44afc1605044b535a7820c1d40e38daa + languageName: node + linkType: hard + "is-plain-obj@npm:^3.0.0": version: 3.0.0 resolution: "is-plain-obj@npm:3.0.0" @@ -11063,6 +11683,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"is-shared-array-buffer@npm:^1.0.2": + version: 1.0.2 + resolution: "is-shared-array-buffer@npm:1.0.2" + dependencies: + call-bind: ^1.0.2 + checksum: 9508929cf14fdc1afc9d61d723c6e8d34f5e117f0bffda4d97e7a5d88c3a8681f633a74f8e3ad1fe92d5113f9b921dc5ca44356492079612f9a247efbce7032a + languageName: node + linkType: hard + "is-stream@npm:^1.1.0": version: 1.1.0 resolution: "is-stream@npm:1.1.0" @@ -11134,6 +11763,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"is-weakref@npm:^1.0.2": + version: 1.0.2 + resolution: "is-weakref@npm:1.0.2" + dependencies: + call-bind: ^1.0.2 + checksum: 95bd9a57cdcb58c63b1c401c60a474b0f45b94719c30f548c891860f051bc2231575c290a6b420c6bc6e7ed99459d424c652bd5bf9a1d5259505dc35b4bf83de + languageName: node + linkType: hard + "is-whitespace-character@npm:^1.0.0": version: 1.0.4 resolution: "is-whitespace-character@npm:1.0.4" @@ -11438,6 +12076,22 @@ fsevents@~2.3.2: languageName: node linkType: hard +"json5@npm:^2.2.0": + version: 2.2.1 + resolution: "json5@npm:2.2.1" + bin: + json5: lib/cli.js + checksum: 74b8a23b102a6f2bf2d224797ae553a75488b5adbaee9c9b6e5ab8b510a2fc6e38f876d4c77dea672d4014a44b2399e15f2051ac2b37b87f74c0c7602003543b + languageName: node + linkType: hard + +"jsonc-parser@npm:^3.0.0": + version: 3.0.0 + resolution: "jsonc-parser@npm:3.0.0" + checksum: 1df2326f1f9688de30c70ff19c5b2a83ba3b89a1036160da79821d1361090775e9db502dc57a67c11b56e1186fc1ed70b887f25c5febf9a3ec4f91435836c99d + languageName: node + linkType: hard + "jsonfile@npm:^4.0.0": version: 4.0.0 resolution: "jsonfile@npm:4.0.0" @@ -11475,6 +12129,16 @@ fsevents@~2.3.2: languageName: node linkType: hard +"jsx-ast-utils@npm:^2.4.1 || ^3.0.0, jsx-ast-utils@npm:^3.3.1": + version: 3.3.1 + resolution: "jsx-ast-utils@npm:3.3.1" + dependencies: + array-includes: ^3.1.5 + object.assign: ^4.1.2 + checksum: 1d4b32fd24bbba561d5ca5c8d6ea095be646f83fc357d6f0cd2752f97f3ba0e0ffabc2f54b37a9d98258fc8ec0e1286cb7723cc1c9dc7af402d74fff72ae0a2b + languageName: node + linkType: hard + "kind-of@npm:^3.0.2, kind-of@npm:^3.0.3, kind-of@npm:^3.2.0": version: 3.2.2 resolution: "kind-of@npm:3.2.2" @@ -11530,6 +12194,22 @@ fsevents@~2.3.2: languageName: node linkType: hard +"language-subtag-registry@npm:~0.3.2": + version: 0.3.21 + resolution: "language-subtag-registry@npm:0.3.21" + checksum: 5f794525a5bfcefeea155a681af1c03365b60e115b688952a53c6e0b9532b09163f57f1fcb69d6150e0e805ec0350644a4cb35da98f4902562915be9f89572a1 + languageName: node + linkType: hard + +"language-tags@npm:^1.0.5": + version: 1.0.5 + resolution: "language-tags@npm:1.0.5" + dependencies: + language-subtag-registry: ~0.3.2 + checksum: c81b5d8b9f5f9cfd06ee71ada6ddfe1cf83044dd5eeefcd1e420ad491944da8957688db4a0a9bc562df4afdc2783425cbbdfd152c01d93179cf86888903123cf + languageName: node + linkType: hard + "lazy-cache@npm:^1.0.3": version: 1.0.4 resolution: "lazy-cache@npm:1.0.4" @@ -13072,7 +13752,7 @@ fsevents@~2.3.2: languageName: node linkType: hard -"minimatch@npm:^3.1.2": +"minimatch@npm:^3.1.1, minimatch@npm:^3.1.2": version: 3.1.2 resolution: "minimatch@npm:3.1.2" dependencies: @@ -13092,13 +13772,6 @@ fsevents@~2.3.2: languageName: node linkType: hard -"minimist@npm:^1.1.1, minimist@npm:^1.2.6": - version: 1.2.6 - resolution: "minimist@npm:1.2.6" - checksum: d15428cd1e11eb14e1233bcfb88ae07ed7a147de251441d61158619dfb32c4d7e9061d09cab4825fdee18ecd6fce323228c8c47b5ba7cd20af378ca4048fb3fb - languageName: node - linkType: hard - "minimist@npm:^1.2.0, minimist@npm:^1.2.5": version: 1.2.5 resolution: "minimist@npm:1.2.5" @@ -13106,6 +13779,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"minimist@npm:^1.2.6": + version: 1.2.6 + resolution: "minimist@npm:1.2.6" + checksum: d15428cd1e11eb14e1233bcfb88ae07ed7a147de251441d61158619dfb32c4d7e9061d09cab4825fdee18ecd6fce323228c8c47b5ba7cd20af378ca4048fb3fb + languageName: node + linkType: hard + "minipass-collect@npm:^1.0.2": version: 1.0.2 resolution: "minipass-collect@npm:1.0.2" @@ -13312,16 +13992,7 @@ fsevents@~2.3.2: languageName: node linkType: hard -"nanoid@npm:^3.3.1": - version: 3.3.2 - resolution: "nanoid@npm:3.3.2" - bin: - nanoid: bin/nanoid.cjs - checksum: 376717f0685251fad77850bd84c6b8d57837c71eeb1c05be7c742140cc1835a5a2953562add05166d6dbc8fb65f3fdffa356213037b967a470e1691dc3e7b9cc - languageName: node - linkType: hard - -"nanoid@npm:^3.3.3": +"nanoid@npm:^3.3.3, nanoid@npm:^3.3.4": version: 3.3.4 resolution: "nanoid@npm:3.3.4" bin: @@ -13474,6 +14145,17 @@ fsevents@~2.3.2: languageName: node linkType: hard +"nice-napi@npm:^1.0.2": + version: 1.0.2 + resolution: "nice-napi@npm:1.0.2" + dependencies: + node-addon-api: ^3.0.0 + node-gyp: latest + node-gyp-build: ^4.2.2 + conditions: "!os=win32" + languageName: node + linkType: hard + "no-case@npm:^3.0.4": version: 3.0.4 resolution: "no-case@npm:3.0.4" @@ -13484,6 +14166,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"node-addon-api@npm:^3.0.0": + version: 3.2.1 + resolution: "node-addon-api@npm:3.2.1" + dependencies: + node-gyp: latest + checksum: 2369986bb0881ccd9ef6bacdf39550e07e089a9c8ede1cbc5fc7712d8e2faa4d50da0e487e333d4125f8c7a616c730131d1091676c9d499af1d74560756b4a18 + languageName: node + linkType: hard + "node-domexception@npm:^1.0.0": version: 1.0.0 resolution: "node-domexception@npm:1.0.0" @@ -13518,6 +14209,17 @@ fsevents@~2.3.2: languageName: node linkType: hard +"node-gyp-build@npm:^4.2.2": + version: 4.5.0 + resolution: "node-gyp-build@npm:4.5.0" + bin: + node-gyp-build: bin.js + node-gyp-build-optional: optional.js + node-gyp-build-test: build-test.js + checksum: d888bae0fb88335f69af1b57a2294a931c5042f36e413d8d364c992c9ebfa0b96ffe773179a5a2c8f04b73856e8634e09cce108dbb9804396d3cc8c5455ff2db + languageName: node + linkType: hard + "node-gyp@npm:latest": version: 8.3.0 resolution: "node-gyp@npm:8.3.0" @@ -13583,10 +14285,10 @@ fsevents@~2.3.2: languageName: node linkType: hard -"node-releases@npm:^2.0.3": - version: 2.0.4 - resolution: "node-releases@npm:2.0.4" - checksum: b32d6c2032c7b169ae3938b416fc50f123f5bd577d54a79b2ae201febf27b22846b01c803dd35ac8689afe840f8ba4e5f7154723db629b80f359836b6707b92f +"node-releases@npm:^2.0.5": + version: 2.0.5 + resolution: "node-releases@npm:2.0.5" + checksum: e85d949addd19f8827f32569d2be5751e7812ccf6cc47879d49f79b5234ff4982225e39a3929315f96370823b070640fb04d79fc0ddec8b515a969a03493a42f languageName: node linkType: hard @@ -13757,6 +14459,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"object-inspect@npm:^1.12.0": + version: 1.12.2 + resolution: "object-inspect@npm:1.12.2" + checksum: a534fc1b8534284ed71f25ce3a496013b7ea030f3d1b77118f6b7b1713829262be9e6243acbcb3ef8c626e2b64186112cb7f6db74e37b2789b9c789ca23048b2 + languageName: node + linkType: hard + "object-keys@npm:^1.0.12, object-keys@npm:^1.1.1": version: 1.1.1 resolution: "object-keys@npm:1.1.1" @@ -13785,6 +14494,28 @@ fsevents@~2.3.2: languageName: node linkType: hard +"object.entries@npm:^1.1.5": + version: 1.1.5 + resolution: "object.entries@npm:1.1.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + es-abstract: ^1.19.1 + checksum: d658696f74fd222060d8428d2a9fda2ce736b700cb06f6bdf4a16a1892d145afb746f453502b2fa55d1dca8ead6f14ddbcf66c545df45adadea757a6c4cd86c7 + languageName: node + linkType: hard + +"object.fromentries@npm:^2.0.5": + version: 2.0.5 + resolution: "object.fromentries@npm:2.0.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + es-abstract: ^1.19.1 + checksum: 61a0b565ded97b76df9e30b569729866e1824cce902f98e90bb106e84f378aea20163366f66dc75c9000e2aad2ed0caf65c6f530cb2abc4c0c0f6c982102db4b + languageName: node + linkType: hard + "object.getownpropertydescriptors@npm:^2.1.0": version: 2.1.3 resolution: "object.getownpropertydescriptors@npm:2.1.3" @@ -13796,6 +14527,16 @@ fsevents@~2.3.2: languageName: node linkType: hard +"object.hasown@npm:^1.1.1": + version: 1.1.1 + resolution: "object.hasown@npm:1.1.1" + dependencies: + define-properties: ^1.1.4 + es-abstract: ^1.19.5 + checksum: d8ed4907ce57f48b93e3b53c418fd6787bf226a51e8d698c91e39b78e80fe5b124cb6282f6a9d5be21cf9e2c7829ab10206dcc6112b7748860eefe641880c793 + languageName: node + linkType: hard + "object.pick@npm:^1.3.0": version: 1.3.0 resolution: "object.pick@npm:1.3.0" @@ -14138,6 +14879,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"pako@npm:^1.0.3, pako@npm:~1.0.5": + version: 1.0.11 + resolution: "pako@npm:1.0.11" + checksum: 1be2bfa1f807608c7538afa15d6f25baa523c30ec870a3228a89579e474a4d992f4293859524e46d5d87fd30fa17c5edf34dbef0671251d9749820b488660b16 + languageName: node + linkType: hard + "pako@npm:~0.2.0": version: 0.2.9 resolution: "pako@npm:0.2.9" @@ -14145,13 +14893,6 @@ fsevents@~2.3.2: languageName: node linkType: hard -"pako@npm:~1.0.5": - version: 1.0.11 - resolution: "pako@npm:1.0.11" - checksum: 1be2bfa1f807608c7538afa15d6f25baa523c30ec870a3228a89579e474a4d992f4293859524e46d5d87fd30fa17c5edf34dbef0671251d9749820b488660b16 - languageName: node - linkType: hard - "parallel-transform@npm:^1.1.0": version: 1.2.0 resolution: "parallel-transform@npm:1.2.0" @@ -14289,6 +15030,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"parse-numeric-range@npm:^1.3.0": + version: 1.3.0 + resolution: "parse-numeric-range@npm:1.3.0" + checksum: 289ca126d5b8ace7325b199218de198014f58ea6895ccc88a5247491d07f0143bf047f80b4a31784f1ca8911762278d7d6ecb90a31dfae31da91cc1a2524c8ce + languageName: node + linkType: hard + "parse5@npm:5.1.0": version: 5.1.0 resolution: "parse5@npm:5.1.0" @@ -14296,6 +15044,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"parse5@npm:^6.0.0": + version: 6.0.1 + resolution: "parse5@npm:6.0.1" + checksum: 7d569a176c5460897f7c8f3377eff640d54132b9be51ae8a8fa4979af940830b2b0c296ce75e5bd8f4041520aadde13170dbdec44889975f906098ea0002f4bd + languageName: node + linkType: hard + "parseurl@npm:~1.3.3": version: 1.3.3 resolution: "parseurl@npm:1.3.3" @@ -14505,6 +15260,21 @@ fsevents@~2.3.2: languageName: node linkType: hard +"piscina@npm:^3.2.0": + version: 3.2.0 + resolution: "piscina@npm:3.2.0" + dependencies: + eventemitter-asyncresource: ^1.0.0 + hdr-histogram-js: ^2.0.1 + hdr-histogram-percentiles-obj: ^3.0.0 + nice-napi: ^1.0.2 + dependenciesMeta: + nice-napi: + optional: true + checksum: c1980c7d45d85f53265652dd2fc62a2b9e9d2321f5bbb9fc1796edb9c1324bb77c153e823a0d6454c3c35098820efedff584737cc282207480afe478a3b8a166 + languageName: node + linkType: hard + "pixrem@npm:^4.0.0": version: 4.0.1 resolution: "pixrem@npm:4.0.1" @@ -14916,6 +15686,19 @@ fsevents@~2.3.2: languageName: node linkType: hard +"postcss-import@npm:^14.1.0": + version: 14.1.0 + resolution: "postcss-import@npm:14.1.0" + dependencies: + postcss-value-parser: ^4.0.0 + read-cache: ^1.0.0 + resolve: ^1.1.7 + peerDependencies: + postcss: ^8.0.0 + checksum: cd45d406e90f67cdab9524352e573cc6b4462b790934a05954e929a6653ebd31288ceebc8ce3c3ed7117ae672d9ebbec57df0bceec0a56e9b259c2e71d47ca86 + languageName: node + linkType: hard + "postcss-initial@npm:^2.0.0": version: 2.0.0 resolution: "postcss-initial@npm:2.0.0" @@ -15410,6 +16193,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"postcss-value-parser@npm:^4.0.0, postcss-value-parser@npm:^4.2.0": + version: 4.2.0 + resolution: "postcss-value-parser@npm:4.2.0" + checksum: 819ffab0c9d51cf0acbabf8996dffbfafbafa57afc0e4c98db88b67f2094cb44488758f06e5da95d7036f19556a4a732525e84289a425f4f6fd8e412a9d7442f + languageName: node + linkType: hard + "postcss-value-parser@npm:^4.0.2": version: 4.1.0 resolution: "postcss-value-parser@npm:4.1.0" @@ -15417,13 +16207,6 @@ fsevents@~2.3.2: languageName: node linkType: hard -"postcss-value-parser@npm:^4.2.0": - version: 4.2.0 - resolution: "postcss-value-parser@npm:4.2.0" - checksum: 819ffab0c9d51cf0acbabf8996dffbfafbafa57afc0e4c98db88b67f2094cb44488758f06e5da95d7036f19556a4a732525e84289a425f4f6fd8e412a9d7442f - languageName: node - linkType: hard - "postcss-values-parser@npm:^1.5.0": version: 1.5.0 resolution: "postcss-values-parser@npm:1.5.0" @@ -15490,25 +16273,25 @@ fsevents@~2.3.2: languageName: node linkType: hard -"postcss@npm:^8.4.12": - version: 8.4.12 - resolution: "postcss@npm:8.4.12" +"postcss@npm:^8.4.13": + version: 8.4.13 + resolution: "postcss@npm:8.4.13" dependencies: - nanoid: ^3.3.1 + nanoid: ^3.3.3 picocolors: ^1.0.0 source-map-js: ^1.0.2 - checksum: 248e3d0f9bbb8efaafcfda7f91627a29bdc9a19f456896886330beb28c5abea0e14c7901b35191928602e2eccbed496b1e94097d27a0b2a980854cd00c7a835f + checksum: 514fb3552805a5d039a2d6b4df3e73f657001716ca93c0d57e6067b0473abdea70276d80afc96005c9aaff82ed5d98062bd97724d3f47ca400fba0b5e9e436ed languageName: node linkType: hard -"postcss@npm:^8.4.13": - version: 8.4.13 - resolution: "postcss@npm:8.4.13" +"postcss@npm:^8.4.14": + version: 8.4.14 + resolution: "postcss@npm:8.4.14" dependencies: - nanoid: ^3.3.3 + nanoid: ^3.3.4 picocolors: ^1.0.0 source-map-js: ^1.0.2 - checksum: 514fb3552805a5d039a2d6b4df3e73f657001716ca93c0d57e6067b0473abdea70276d80afc96005c9aaff82ed5d98062bd97724d3f47ca400fba0b5e9e436ed + checksum: fe58766ff32e4becf65a7d57678995cfd239df6deed2fe0557f038b47c94e4132e7e5f68b5aa820c13adfec32e523b693efaeb65798efb995ce49ccd83953816 languageName: node linkType: hard @@ -16648,6 +17431,17 @@ fsevents@~2.3.2: languageName: node linkType: hard +"regexp.prototype.flags@npm:^1.4.1, regexp.prototype.flags@npm:^1.4.3": + version: 1.4.3 + resolution: "regexp.prototype.flags@npm:1.4.3" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + functions-have-names: ^1.2.2 + checksum: 51228bae732592adb3ededd5e15426be25f289e9c4ef15212f4da73f4ec3919b6140806374b8894036a86020d054a8d2657d3fee6bb9b4d35d8939c20030b7a6 + languageName: node + linkType: hard + "regexpp@npm:^3.2.0": version: 3.2.0 resolution: "regexpp@npm:3.2.0" @@ -16700,6 +17494,16 @@ fsevents@~2.3.2: languageName: node linkType: hard +"rehype-parse@npm:^7.0.1": + version: 7.0.1 + resolution: "rehype-parse@npm:7.0.1" + dependencies: + hast-util-from-parse5: ^6.0.0 + parse5: ^6.0.0 + checksum: c3c914aa9281853290eff6b09e0bed6843934e788b957e25219e91f0bf244a183d2f5e042c7d21543276571f9b49a6bae90f4640b8f885f2773392ffa57baf4b + languageName: node + linkType: hard + "rehype-stringify@npm:^9.0.3": version: 9.0.3 resolution: "rehype-stringify@npm:9.0.3" @@ -16816,7 +17620,7 @@ fsevents@~2.3.2: languageName: node linkType: hard -"repeat-string@npm:^1.5.2, repeat-string@npm:^1.5.4, repeat-string@npm:^1.6.1": +"repeat-string@npm:^1.0.0, repeat-string@npm:^1.5.2, repeat-string@npm:^1.5.4, repeat-string@npm:^1.6.1": version: 1.6.1 resolution: "repeat-string@npm:1.6.1" checksum: 1b809fc6db97decdc68f5b12c4d1a671c8e3f65ec4a40c238bc5200e44e85bcc52a54f78268ab9c29fcf5fe4f1343e805420056d1f30fa9a9ee4c2d93e3cc6c0 @@ -16985,6 +17789,19 @@ fsevents@~2.3.2: languageName: node linkType: hard +"resolve@npm:^2.0.0-next.3": + version: 2.0.0-next.4 + resolution: "resolve@npm:2.0.0-next.4" + dependencies: + is-core-module: ^2.9.0 + path-parse: ^1.0.7 + supports-preserve-symlinks-flag: ^1.0.0 + bin: + resolve: bin/resolve + checksum: c438ac9a650f2030fd074219d7f12ceb983b475da2d89ad3d6dd05fbf6b7a0a8cd37d4d10b43cb1f632bc19f22246ab7f36ebda54d84a29bfb2910a0680906d3 + languageName: node + linkType: hard + "resolve@patch:resolve@^1.1.6#~builtin, resolve@patch:resolve@^1.1.7#~builtin, resolve@patch:resolve@^1.10.0#~builtin, resolve@patch:resolve@^1.14.2#~builtin, resolve@patch:resolve@^1.19.0#~builtin, resolve@patch:resolve@^1.20.0#~builtin, resolve@patch:resolve@^1.3.3#~builtin": version: 1.20.0 resolution: "resolve@patch:resolve@npm%3A1.20.0#~builtin::version=1.20.0&hash=07638b" @@ -17008,6 +17825,19 @@ fsevents@~2.3.2: languageName: node linkType: hard +"resolve@patch:resolve@^2.0.0-next.3#~builtin": + version: 2.0.0-next.4 + resolution: "resolve@patch:resolve@npm%3A2.0.0-next.4#~builtin::version=2.0.0-next.4&hash=07638b" + dependencies: + is-core-module: ^2.9.0 + path-parse: ^1.0.7 + supports-preserve-symlinks-flag: ^1.0.0 + bin: + resolve: bin/resolve + checksum: 4bf9f4f8a458607af90518ff73c67a4bc1a38b5a23fef2bb0ccbd45e8be89820a1639b637b0ba377eb2be9eedfb1739a84cde24fe4cd670c8207d8fea922b011 + languageName: node + linkType: hard + "restore-cursor@npm:^2.0.0": version: 2.0.0 resolution: "restore-cursor@npm:2.0.0" @@ -17243,6 +18073,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"sanitize-xml-string@npm:^1.1.0": + version: 1.1.0 + resolution: "sanitize-xml-string@npm:1.1.0" + checksum: 30272985794e1d7a3e9f04e211f712d02be037dc7c4b1c594dfa5f3151fc5ff2913a2650a06ecd2b1d7181f156096faddbc36728b82ad3722972e0200ac40669 + languageName: node + linkType: hard + "sanity-diff-patch@npm:^1.0.9": version: 1.0.9 resolution: "sanity-diff-patch@npm:1.0.9" @@ -17559,6 +18396,28 @@ fsevents@~2.3.2: languageName: node linkType: hard +"shiki-renderer-hast@npm:^1.1.5": + version: 1.1.5 + resolution: "shiki-renderer-hast@npm:1.1.5" + dependencies: + hastscript: ^7.0.2 + peerDependencies: + shiki: ^0.9 || ^0.10.0 + checksum: 73919ab12a64c1728c068ccb77150decf1e6bd9e333720b517393fc84e78ed4fccaf81e5c2a16685f5f6cd9aa3dbe68d99fb10657a79a0ebda061a497bbf1b21 + languageName: node + linkType: hard + +"shiki@npm:^0.9.4": + version: 0.9.15 + resolution: "shiki@npm:0.9.15" + dependencies: + jsonc-parser: ^3.0.0 + vscode-oniguruma: ^1.6.1 + vscode-textmate: 5.2.0 + checksum: 58d1e3e106320252b67c63dc1269c4b834152e9c675a06a4565ec41db1c93aea2dd94e22640d7ec99334cb47cd41b914642d936577143b689ef2a0db7d938c13 + languageName: node + linkType: hard + "side-channel@npm:^1.0.4": version: 1.0.4 resolution: "side-channel@npm:1.0.4" @@ -18052,6 +18911,22 @@ fsevents@~2.3.2: languageName: node linkType: hard +"string.prototype.matchall@npm:^4.0.7": + version: 4.0.7 + resolution: "string.prototype.matchall@npm:4.0.7" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.3 + es-abstract: ^1.19.1 + get-intrinsic: ^1.1.1 + has-symbols: ^1.0.3 + internal-slot: ^1.0.3 + regexp.prototype.flags: ^1.4.1 + side-channel: ^1.0.4 + checksum: fc09f3ccbfb325de0472bcc87a6be0598a7499e0b4a31db5789676155b15754a4cc4bb83924f15fc9ed48934dac7366ee52c8b9bd160bed6fd072c93b489e75c + languageName: node + linkType: hard + "string.prototype.trimend@npm:^1.0.4": version: 1.0.4 resolution: "string.prototype.trimend@npm:1.0.4" @@ -18062,6 +18937,17 @@ fsevents@~2.3.2: languageName: node linkType: hard +"string.prototype.trimend@npm:^1.0.5": + version: 1.0.5 + resolution: "string.prototype.trimend@npm:1.0.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.4 + es-abstract: ^1.19.5 + checksum: d44f543833112f57224e79182debadc9f4f3bf9d48a0414d6f0cbd2a86f2b3e8c0ca1f95c3f8e5b32ae83e91554d79d932fc746b411895f03f93d89ed3dfb6bc + languageName: node + linkType: hard + "string.prototype.trimstart@npm:^1.0.4": version: 1.0.4 resolution: "string.prototype.trimstart@npm:1.0.4" @@ -18072,6 +18958,17 @@ fsevents@~2.3.2: languageName: node linkType: hard +"string.prototype.trimstart@npm:^1.0.5": + version: 1.0.5 + resolution: "string.prototype.trimstart@npm:1.0.5" + dependencies: + call-bind: ^1.0.2 + define-properties: ^1.1.4 + es-abstract: ^1.19.5 + checksum: a4857c5399ad709d159a77371eeaa8f9cc284469a0b5e1bfe405de16f1fd4166a8ea6f4180e55032f348d1b679b1599fd4301fbc7a8b72bdb3e795e43f7b1048 + languageName: node + linkType: hard + "string_decoder@npm:^1.0.0, string_decoder@npm:^1.1.1": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" @@ -18329,13 +19226,13 @@ fsevents@~2.3.2: linkType: hard "tailwindcss@npm:^3.0.24": - version: 3.0.24 - resolution: "tailwindcss@npm:3.0.24" + version: 3.1.4 + resolution: "tailwindcss@npm:3.1.4" dependencies: - arg: ^5.0.1 + arg: ^5.0.2 chokidar: ^3.5.3 color-name: ^1.1.4 - detective: ^5.2.0 + detective: ^5.2.1 didyoumean: ^1.2.2 dlv: ^1.1.3 fast-glob: ^3.2.11 @@ -18345,7 +19242,8 @@ fsevents@~2.3.2: normalize-path: ^3.0.0 object-hash: ^3.0.0 picocolors: ^1.0.0 - postcss: ^8.4.12 + postcss: ^8.4.14 + postcss-import: ^14.1.0 postcss-js: ^4.0.0 postcss-load-config: ^3.1.4 postcss-nested: 5.0.6 @@ -18358,7 +19256,7 @@ fsevents@~2.3.2: bin: tailwind: lib/cli.js tailwindcss: lib/cli.js - checksum: 52a21192b70ab90678d6cec24ca6f93b3a396599e2d842f6077b670be14e577b1e3fbae8776e64505d383118746287353ed99d2a047258254f4ce3879b996b58 + checksum: 716b7d80e51c4904d845920cbb88483114ae6f14b20328b9206f49fd642bbcd390b9764fc574f85901b0377184c5d4eb94c00bcf73e280f81e772cf621d390a5 languageName: node linkType: hard @@ -19066,6 +19964,18 @@ fsevents@~2.3.2: languageName: node linkType: hard +"unbox-primitive@npm:^1.0.2": + version: 1.0.2 + resolution: "unbox-primitive@npm:1.0.2" + dependencies: + call-bind: ^1.0.2 + has-bigints: ^1.0.2 + has-symbols: ^1.0.3 + which-boxed-primitive: ^1.0.2 + checksum: b7a1cf5862b5e4b5deb091672ffa579aa274f648410009c81cca63fed3b62b610c4f3b773f912ce545bb4e31edc3138975b5bc777fc6e4817dca51affb6380e9 + languageName: node + linkType: hard + "unherit@npm:^1.0.4": version: 1.1.3 resolution: "unherit@npm:1.1.3" @@ -19122,7 +20032,7 @@ fsevents@~2.3.2: languageName: node linkType: hard -"unified@npm:^10.1.2": +"unified@npm:^10.1.0, unified@npm:^10.1.2": version: 10.1.2 resolution: "unified@npm:10.1.2" dependencies: @@ -19151,6 +20061,20 @@ fsevents@~2.3.2: languageName: node linkType: hard +"unified@npm:^9.2.0": + version: 9.2.2 + resolution: "unified@npm:9.2.2" + dependencies: + bail: ^1.0.0 + extend: ^3.0.0 + is-buffer: ^2.0.0 + is-plain-obj: ^2.0.0 + trough: ^1.0.0 + vfile: ^4.0.0 + checksum: 7c24461be7de4145939739ce50d18227c5fbdf9b3bc5a29dabb1ce26dd3e8bd4a1c385865f6f825f3b49230953ee8b591f23beab3bb3643e3e9dc37aa8a089d5 + languageName: node + linkType: hard + "union-value@npm:^1.0.0": version: 1.0.1 resolution: "union-value@npm:1.0.1" @@ -19231,6 +20155,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"unist-util-find-after@npm:^3.0.0": + version: 3.0.0 + resolution: "unist-util-find-after@npm:3.0.0" + dependencies: + unist-util-is: ^4.0.0 + checksum: daa9a28f6cdf533a72ce7ec4864dbe0f11f0fd3efd337b54c08a8a9a47cdc8d10a299cd984d7f512a57e97af012df052210a51aab7c9afd6b1e24da3b2d0a714 + languageName: node + linkType: hard + "unist-util-find-after@npm:^4.0.0": version: 4.0.0 resolution: "unist-util-find-after@npm:4.0.0" @@ -19269,6 +20202,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"unist-util-modify-children@npm:^2.0.0": + version: 2.0.0 + resolution: "unist-util-modify-children@npm:2.0.0" + dependencies: + array-iterate: ^1.0.0 + checksum: 7c8e11c320e2c8f8e0f7ab32a0d5a88317a8ed40c30ef0dca1038252eae9ca31db7e24f3c77799ae086bf1f73ee8cc34056e12334b05da304287e3a5b8700034 + languageName: node + linkType: hard + "unist-util-position-from-estree@npm:^1.0.0, unist-util-position-from-estree@npm:^1.1.0": version: 1.1.1 resolution: "unist-util-position-from-estree@npm:1.1.1" @@ -19311,6 +20253,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"unist-util-stringify-position@npm:^2.0.0": + version: 2.0.3 + resolution: "unist-util-stringify-position@npm:2.0.3" + dependencies: + "@types/unist": ^2.0.2 + checksum: f755cadc959f9074fe999578a1a242761296705a7fe87f333a37c00044de74ab4b184b3812989a57d4cd12211f0b14ad397b327c3a594c7af84361b1c25a7f09 + languageName: node + linkType: hard + "unist-util-stringify-position@npm:^3.0.0": version: 3.0.0 resolution: "unist-util-stringify-position@npm:3.0.0" @@ -19459,6 +20410,20 @@ fsevents@~2.3.2: languageName: node linkType: hard +"update-browserslist-db@npm:^1.0.4": + version: 1.0.4 + resolution: "update-browserslist-db@npm:1.0.4" + dependencies: + escalade: ^3.1.1 + picocolors: ^1.0.0 + peerDependencies: + browserslist: ">= 4.21.0" + bin: + browserslist-lint: cli.js + checksum: 7c7da28d0fc733b17e01c8fa9385ab909eadce64b8ea644e9603867dc368c2e2a6611af8247e72612b23f9e7cb87ac7c7585a05ff94e1759e9d646cbe9bf49a7 + languageName: node + linkType: hard + "uri-js@npm:^4.2.2": version: 4.4.1 resolution: "uri-js@npm:4.4.1" @@ -19729,6 +20694,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"vfile-location@npm:^3.2.0": + version: 3.2.0 + resolution: "vfile-location@npm:3.2.0" + checksum: 9bb3df6d0be31b5dd2d8da0170c27b7045c64493a8ba7b6ff7af8596c524fc8896924b8dd85ae12d201eead2709217a0fbc44927b7264f4bbf0aa8027a78be9c + languageName: node + linkType: hard + "vfile-message@npm:^1.0.0": version: 1.1.1 resolution: "vfile-message@npm:1.1.1" @@ -19738,6 +20710,16 @@ fsevents@~2.3.2: languageName: node linkType: hard +"vfile-message@npm:^2.0.0": + version: 2.0.4 + resolution: "vfile-message@npm:2.0.4" + dependencies: + "@types/unist": ^2.0.0 + unist-util-stringify-position: ^2.0.0 + checksum: 1bade499790f46ca5aba04bdce07a1e37c2636a8872e05cf32c26becc912826710b7eb063d30c5754fdfaeedc8a7658e78df10b3bc535c844890ec8a184f5643 + languageName: node + linkType: hard + "vfile-message@npm:^3.0.0": version: 3.0.2 resolution: "vfile-message@npm:3.0.2" @@ -19760,6 +20742,18 @@ fsevents@~2.3.2: languageName: node linkType: hard +"vfile@npm:^4.0.0": + version: 4.2.1 + resolution: "vfile@npm:4.2.1" + dependencies: + "@types/unist": ^2.0.0 + is-buffer: ^2.0.0 + unist-util-stringify-position: ^2.0.0 + vfile-message: ^2.0.0 + checksum: ee5726e10d170472cde778fc22e0f7499caa096eb85babea5d0ce0941455b721037ee1c9e6ae506ca2803250acd313d0f464328ead0b55cfe7cb6315f1b462d6 + languageName: node + linkType: hard + "vfile@npm:^5.0.0": version: 5.2.0 resolution: "vfile@npm:5.2.0" @@ -19846,6 +20840,20 @@ fsevents@~2.3.2: languageName: node linkType: hard +"vscode-oniguruma@npm:^1.6.1": + version: 1.6.2 + resolution: "vscode-oniguruma@npm:1.6.2" + checksum: 6b754acdafd5b68242ea5938bb00a32effc16c77f471d4f0f337d879d0e8e592622998e2441f42d9a7ff799c1593f31c11f26ca8d9bf9917e3ca881d3c1f3e19 + languageName: node + linkType: hard + +"vscode-textmate@npm:5.2.0": + version: 5.2.0 + resolution: "vscode-textmate@npm:5.2.0" + checksum: 5449b42d451080f6f3649b66948f4b5ee4643c4e88cfe3558a3b31c84c78060cfdd288c4958c1690eaa5cd65d09992fa6b7c3bef9d4aa72b3651054a04624d20 + languageName: node + linkType: hard + "w3c-hr-time@npm:^1.0.1": version: 1.0.2 resolution: "w3c-hr-time@npm:1.0.2" @@ -19899,6 +20907,13 @@ fsevents@~2.3.2: languageName: node linkType: hard +"web-namespaces@npm:^1.0.0": + version: 1.1.4 + resolution: "web-namespaces@npm:1.1.4" + checksum: 5149842ccbfbc56fe4f8758957b3f8c8616a281874a5bb84aa1b305e4436a9bad853d21c629a7b8f174902449e1489c7a6c724fccf60965077c5636bd8aed42b + languageName: node + linkType: hard + "web-streams-polyfill@npm:^3.0.3": version: 3.2.1 resolution: "web-streams-polyfill@npm:3.2.1" From 71f286c7448f6433eee37e798c934bc5e6e9e045 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Thu, 7 Jul 2022 12:23:36 +0200 Subject: [PATCH 02/13] docs: test parallelism changes with gatsby docs --- .eslintrc | 2 +- examples/gatsbydocs/.gitignore | 4 ++ examples/gatsbydocs/contentlayer.config.ts | 80 ++++++++++++++++++++++ examples/gatsbydocs/next-env.d.ts | 5 ++ examples/gatsbydocs/next.config.js | 4 ++ examples/gatsbydocs/package.json | 25 +++++++ examples/gatsbydocs/pages/[...id].tsx | 28 ++++++++ examples/gatsbydocs/pages/index.tsx | 22 ++++++ examples/gatsbydocs/tsconfig.json | 41 +++++++++++ 9 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 examples/gatsbydocs/.gitignore create mode 100644 examples/gatsbydocs/contentlayer.config.ts create mode 100644 examples/gatsbydocs/next-env.d.ts create mode 100644 examples/gatsbydocs/next.config.js create mode 100644 examples/gatsbydocs/package.json create mode 100644 examples/gatsbydocs/pages/[...id].tsx create mode 100644 examples/gatsbydocs/pages/index.tsx create mode 100644 examples/gatsbydocs/tsconfig.json diff --git a/.eslintrc b/.eslintrc index 52b9e7b4..e8dc0da9 100644 --- a/.eslintrc +++ b/.eslintrc @@ -4,7 +4,7 @@ "node": true, "es6": true }, - "ignorePatterns": ["packages/_archive/*", "examples/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"], + "ignorePatterns": ["packages/_archive/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"], "parser": "@typescript-eslint/parser", "plugins": ["@typescript-eslint", "simple-import-sort", "import"], "extends": ["plugin:react-hooks/recommended", "plugin:@typescript-eslint/recommended", "prettier"], diff --git a/examples/gatsbydocs/.gitignore b/examples/gatsbydocs/.gitignore new file mode 100644 index 00000000..33ec38aa --- /dev/null +++ b/examples/gatsbydocs/.gitignore @@ -0,0 +1,4 @@ +content +.next +.contentlayer +node_modules diff --git a/examples/gatsbydocs/contentlayer.config.ts b/examples/gatsbydocs/contentlayer.config.ts new file mode 100644 index 00000000..9d86bfeb --- /dev/null +++ b/examples/gatsbydocs/contentlayer.config.ts @@ -0,0 +1,80 @@ +import rehypeShiki from '@stefanprobst/rehype-shiki' +import type { FieldDef} from 'contentlayer/source-files'; +import { defineDocumentType, defineNestedType,makeSource } from 'contentlayer/source-files' +import { createRequire } from 'module' +import * as path from 'path' +import * as shiki from 'shiki' + +const Example = defineNestedType(() => ({ + name: 'Example', + fields: { + label: { type: 'string' }, + href: { type: 'string' }, + }, +})) + +const fields: Record = { + title: { + type: 'string', + required: true, + }, + jsdoc: { + type: 'list', + of: {type: 'string'}, + }, + tableOfContentsDepth: { type: 'number' }, + showTopLevelSignatures: { type: 'boolean' }, + date: { type: 'string' }, + version: { type: 'string' }, + canonicalLink: { type: 'string' }, + apiCalls: { type: 'string' }, + contentsHeading: { type: 'string' }, + description: { type: 'string' }, + examples: { + type: 'list', + of: Example, + }, +} + +const Reference = defineDocumentType(() => ({ + name: 'Reference', + filePathPattern: 'docs/reference/**/*.md', + fields, +})) + +const HowTo = defineDocumentType(() => ({ + name: 'HowTo', + filePathPattern: 'docs/how-to/**/*.md', + fields, +})) + +const Conceptual = defineDocumentType(() => ({ + name: 'Conceptual', + filePathPattern: 'docs/conceptual/**/*.md', + fields, +})) + +const Tutorial = defineDocumentType(() => ({ + name: 'Tutorial', + filePathPattern: 'tutorial/**/*.md', + fields, +})) + +export default makeSource(async () => { + const require = createRequire(import.meta.url) + const shikiPkgPath = (dir: string) => path.join(require.resolve('shiki'), '..', '..', dir, path.sep) + const highlighter = await shiki.getHighlighter({ + paths: { languages: shikiPkgPath('languages'), themes: shikiPkgPath('themes') }, + theme: 'github-light', + }) + + return { + contentDirPath: 'content', + documentTypes: [Reference, HowTo, Conceptual, Tutorial], + // onUnknownDocuments: 'skip-ignore', + markdown: { + // '@stefanprobst/rehype-shiki', {} + rehypePlugins: [[rehypeShiki, { highlighter }]], + }, + } +}) diff --git a/examples/gatsbydocs/next-env.d.ts b/examples/gatsbydocs/next-env.d.ts new file mode 100644 index 00000000..4f11a03d --- /dev/null +++ b/examples/gatsbydocs/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/examples/gatsbydocs/next.config.js b/examples/gatsbydocs/next.config.js new file mode 100644 index 00000000..f92aa8bb --- /dev/null +++ b/examples/gatsbydocs/next.config.js @@ -0,0 +1,4 @@ +const { withContentlayer } = require('next-contentlayer') + +// module.exports = {} +module.exports = withContentlayer({}) diff --git a/examples/gatsbydocs/package.json b/examples/gatsbydocs/package.json new file mode 100644 index 00000000..3c73bd66 --- /dev/null +++ b/examples/gatsbydocs/package.json @@ -0,0 +1,25 @@ +{ + "name": "examples-gatsby-docs", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "12.1.6", + "react": "18.1.0", + "react-dom": "18.1.0" + }, + "devDependencies": { + "@leafac/rehype-shiki": "^1.3.1", + "@stefanprobst/rehype-shiki": "^2.0.4", + "@types/react": "18.0.9", + "@types/react-dom": "^17.0.9", + "contentlayer": "workspace:*", + "eslint-config-next": "^11.0.1", + "next-contentlayer": "workspace:*", + "shiki": "^0.9.4", + "typescript": "4.6.4" + } +} diff --git a/examples/gatsbydocs/pages/[...id].tsx b/examples/gatsbydocs/pages/[...id].tsx new file mode 100644 index 00000000..c01a360a --- /dev/null +++ b/examples/gatsbydocs/pages/[...id].tsx @@ -0,0 +1,28 @@ +import Head from 'next/head' +import type { FC } from 'react' + +import { allDocuments } from 'contentlayer/generated' +import type { DocumentTypes } from 'contentlayer/generated' + +export const getStaticPaths = () => { + const paths = allDocuments.map((_) => `/${_._raw.flattenedPath}`) + + return { paths, fallback: false } +} + +export const getStaticProps = (context: any) => { + const doc = allDocuments.find((_) => _._raw.flattenedPath === context.params.id.join('/')) + + return { props: { doc } } +} + +const Page: FC<{ doc: DocumentTypes }> = ({ doc }) => ( + <> + + {doc.title} + +
+ +) + +export default Page diff --git a/examples/gatsbydocs/pages/index.tsx b/examples/gatsbydocs/pages/index.tsx new file mode 100644 index 00000000..277c375c --- /dev/null +++ b/examples/gatsbydocs/pages/index.tsx @@ -0,0 +1,22 @@ +import type { InferGetStaticPropsType } from 'next' +import type { FC } from 'react' + +import { allDocuments } from 'contentlayer/generated' + +export const getStaticProps = () => { + const docs = allDocuments.map((_) => ({ path: _._raw.flattenedPath, title: _.title })) + + return { props: { docs } } +} + +const Page: FC> = ({ docs }) => ( +
+ {docs.map((doc) => ( + + {doc.title} + + ))} +
+) + +export default Page diff --git a/examples/gatsbydocs/tsconfig.json b/examples/gatsbydocs/tsconfig.json new file mode 100644 index 00000000..1c5b537a --- /dev/null +++ b/examples/gatsbydocs/tsconfig.json @@ -0,0 +1,41 @@ +{ + "compilerOptions": { + "jsx": "preserve", + "allowSyntheticDefaultImports": true, + "rootDir": ".", + "allowJs": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "incremental": false, + "composite": false, + "noEmit": true, + "target": "es5", + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], + "strict": false, + "module": "esnext", + "moduleResolution": "node", + "isolatedModules": true, + "baseUrl": ".", + "paths": { + "contentlayer/generated": [ + "./.contentlayer/generated" + ] + } + }, + "include": [ + "next-env.d.ts", + "**/*.tsx", + "**/*.ts", + ".contentlayer/generated" + ], + "exclude": [ + "node_modules", + "gatsby" + ] +} From aa764e961fe94ee11bd373760b2050e75af021aa Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Thu, 7 Jul 2022 12:24:00 +0200 Subject: [PATCH 03/13] docs: add TODO --- packages/@contentlayer/utils/src/effect/Stream.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@contentlayer/utils/src/effect/Stream.ts b/packages/@contentlayer/utils/src/effect/Stream.ts index 4e5d77f7..078fc729 100644 --- a/packages/@contentlayer/utils/src/effect/Stream.ts +++ b/packages/@contentlayer/utils/src/effect/Stream.ts @@ -65,6 +65,8 @@ export const startWith = export const startWithRight = (value: A2) => (stream: S.Stream>): S.Stream> => + // TODO: Is this correct? Technically, I think this inserts the "init" _after_ any events -> race condition? + // how about `S.merge_(S.fromIterable([E.right(value)]), stream)` S.merge_(stream, S.fromIterable([E.right(value)])) export const chainMapEitherRight = From a0bff262c86f0f5894c59f4304565b18bd7b06e3 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Thu, 7 Jul 2022 14:51:24 +0200 Subject: [PATCH 04/13] feat(@contentlayer): serialize DocumentTypeMapState --- examples/gatsbydocs/pages/[...id].tsx | 5 +- examples/gatsbydocs/pages/index.tsx | 3 +- .../src/fetchData/DocumentTypeMap.ts | 46 ++- .../src/fetchData/fetchAllDocuments.ts | 329 ++++++++++-------- .../makeCacheItemFromFilePath.worker.ts | 12 +- 5 files changed, 232 insertions(+), 163 deletions(-) diff --git a/examples/gatsbydocs/pages/[...id].tsx b/examples/gatsbydocs/pages/[...id].tsx index c01a360a..d3ea213f 100644 --- a/examples/gatsbydocs/pages/[...id].tsx +++ b/examples/gatsbydocs/pages/[...id].tsx @@ -1,9 +1,8 @@ +import type { DocumentTypes } from 'contentlayer/generated' +import { allDocuments } from 'contentlayer/generated' import Head from 'next/head' import type { FC } from 'react' -import { allDocuments } from 'contentlayer/generated' -import type { DocumentTypes } from 'contentlayer/generated' - export const getStaticPaths = () => { const paths = allDocuments.map((_) => `/${_._raw.flattenedPath}`) diff --git a/examples/gatsbydocs/pages/index.tsx b/examples/gatsbydocs/pages/index.tsx index 277c375c..47ba9ea8 100644 --- a/examples/gatsbydocs/pages/index.tsx +++ b/examples/gatsbydocs/pages/index.tsx @@ -1,8 +1,7 @@ +import { allDocuments } from 'contentlayer/generated' import type { InferGetStaticPropsType } from 'next' import type { FC } from 'react' -import { allDocuments } from 'contentlayer/generated' - export const getStaticProps = () => { const docs = allDocuments.map((_) => ({ path: _._raw.flattenedPath, title: _.title })) diff --git a/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts b/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts index 06938188..fc82f916 100644 --- a/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts +++ b/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts @@ -2,33 +2,47 @@ import type { PosixFilePath } from '@contentlayer/utils' import type { Has } from '@contentlayer/utils/effect' import { HashMap, O, pipe, State, T, Tagged } from '@contentlayer/utils/effect' -type DocumentTypeName = string +export type DocumentTypeName = string export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{ - readonly map: HashMap.HashMap + readonly map: HashMap.HashMap }> { - static init = () => new DocumentTypeMap({ map: HashMap.make() }) + static init = () => new DocumentTypeMap({ map: HashMap.make() }) - add = (documentTypeName: DocumentTypeName, filePath: PosixFilePath) => { - const oldPaths = pipe( - HashMap.get_(this.map, documentTypeName), - O.getOrElse(() => []), - ) + add = (documentTypeName: DocumentTypeName, filePath: PosixFilePath) => { + const oldPaths = pipe( + HashMap.get_(this.map, documentTypeName), + O.getOrElse(() => []), + ) - return new DocumentTypeMap({ - map: HashMap.set_(this.map, documentTypeName, [...oldPaths, filePath]), - }) - } + return new DocumentTypeMap({ + map: HashMap.set_(this.map, documentTypeName, [...oldPaths, filePath]), + }) + } - getFilePaths = (documentTypeName: DocumentTypeName): O.Option => - HashMap.get_(this.map, documentTypeName) + getFilePaths = (documentTypeName: DocumentTypeName): O.Option => + HashMap.get_(this.map, documentTypeName) } /** - * This state is needed for certain kinds of error handling (e.g. to check if singleton documents have been provided) - */ +* This state is needed for certain kinds of error handling (e.g. to check if singleton documents have been provided) +*/ export const DocumentTypeMapState = State.State(DocumentTypeMap._tag) +export const serialize = () => pipe( + DocumentTypeMapState.get, + // FIXME: unsafe + T.chain(map => T.succeedWith(() => JSON.stringify(map))), +); + +export function provideFromSerialized(serialized: string) { + return pipe( + // FIXME: unsafe + T.succeedWith(() => JSON.parse(serialized)), + T.map(map => T.provideSomeLayer(DocumentTypeMapState.Live(new DocumentTypeMap(map)))) + ) +} + export const provideDocumentTypeMapState = T.provideSomeLayer(DocumentTypeMapState.Live(DocumentTypeMap.init())) export type HasDocumentTypeMapState = Has> diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index db973947..c7eb6cf5 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -1,9 +1,10 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' import { asMutableArray, posixFilePath } from '@contentlayer/utils' -import type { HasConsole } from '@contentlayer/utils/effect' -import { Chunk, O, OT, pipe, T } from '@contentlayer/utils/effect' +import type { HasConsole} from '@contentlayer/utils/effect'; +import { Chunk, HashMap , O, OT, pipe, T } from '@contentlayer/utils/effect' import { fs } from '@contentlayer/utils/node' +import { Has } from '@effect-ts/core'; import glob from 'fast-glob' import * as os from 'node:os' @@ -12,152 +13,206 @@ import type { Flags } from '../index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' import { DocumentTypeMapState, provideDocumentTypeMapState } from './DocumentTypeMap.js' +import * as SF from './DocumentTypeMap.js' import { makeCacheItemFromFilePath } from './makeCacheItemFromFilePath.js' -import {createPool} from './makeCacheItemFromFilePath.worker.js' +// import {createPool} from './makeCacheItemFromFilePath.worker.js' export const fetchAllDocuments = ({ - coreSchemaDef, - filePathPatternMap, - contentDirPath, - contentDirInclude, - contentDirExclude, - contentTypeMap, - flags, - options, - previousCache, - verbose, + coreSchemaDef, + filePathPatternMap, + contentDirPath, + contentDirInclude, + contentDirExclude, + contentTypeMap, + flags, + options, + previousCache, + verbose, }: { - coreSchemaDef: core.SchemaDef - filePathPatternMap: FilePathPatternMap - contentDirPath: PosixFilePath - contentDirInclude: readonly PosixFilePath[] - contentDirExclude: readonly PosixFilePath[] - contentTypeMap: ContentTypeMap - flags: Flags - options: core.PluginOptions - previousCache: core.DataCache.Cache | undefined - verbose: boolean -}): T.Effect => - pipe( - T.gen(function* ($) { - const allRelativeFilePaths = yield* $( - getAllRelativeFilePaths({ contentDirPath, contentDirInclude, contentDirExclude }), - ) - - const concurrencyLimit = os.cpus().length - - const run = createPool(); - // TODO: deconstruct environment state - const environmentSeed = {}; - - const { dataErrors, documents } = yield* $( - pipe( - allRelativeFilePaths, - // TODO: parallalize - T.forEachParN(concurrencyLimit, (relativeFilePath) => - run({environmentSeed, input: { - relativeFilePath, - filePathPatternMap, - coreSchemaDef, - contentDirPath, - options, - previousCache, - contentTypeMap, - }}), - ), - T.map(Chunk.partitionThese), - T.map(({ tuple: [errors, docs] }) => ({ dataErrors: Chunk.toArray(errors), documents: Chunk.toArray(docs) })), - ), - ) - - const singletonDataErrors = yield* $(validateSingletonDocuments({ coreSchemaDef, filePathPatternMap })) - - yield* $( - FetchDataError.handleErrors({ - errors: [...dataErrors, ...singletonDataErrors], - documentCount: allRelativeFilePaths.length, - flags, - options, - schemaDef: coreSchemaDef, - contentDirPath, - verbose, + coreSchemaDef: core.SchemaDef + filePathPatternMap: FilePathPatternMap + contentDirPath: PosixFilePath + contentDirInclude: readonly PosixFilePath[] + contentDirExclude: readonly PosixFilePath[] + contentTypeMap: ContentTypeMap + flags: Flags + options: core.PluginOptions + previousCache: core.DataCache.Cache | undefined + verbose: boolean + }): T.Effect => + pipe( + T.gen(function* ($) { + const allRelativeFilePaths = yield* $( + getAllRelativeFilePaths({ contentDirPath, contentDirInclude, contentDirExclude }), + ) + + const concurrencyLimit = os.cpus().length + + // host -> worker + const serialized = yield* $(pipe( + SF.DocumentTypeMapState.update(_ => _.add('foo', posixFilePath('bar'))), + T.chain(() => DocumentTypeMapState.get), + T.tap(map => { + console.log('start', map); + return T.unit; + }), + T.chain(map => T.succeedWith(() => { + const r: Record = {}; + for (const [k, v] of map.map) { + r[k] = v; + } + return JSON.stringify(r); + })), + T.tap(serialized => { + console.log('serialized', serialized); + return T.unit; + }), + SF.provideDocumentTypeMapState, + )); + + // worker input + const layer = yield* $(pipe( + T.succeedWith(() => { + const i: Record = JSON.parse(serialized); + return Object.entries(i).reduce((map, [key, value]) => { + return HashMap.set(key, value)(map) + }, HashMap.make()); + }), + T.tap(deserialized => { + console.log('deserialized', deserialized); + return T.unit; + }), + T.map(map => new SF.DocumentTypeMap({map})), + T.tap(instantiated => { + console.log('instantiated', instantiated); + return T.unit; + }), + )); + + // worker -> host + yield* $(pipe( + SF.DocumentTypeMapState.update(_ => _.add('qux', posixFilePath('baz'))), + T.chain(() => SF.DocumentTypeMapState.get), + T.tap(end => { + console.log('end', end); + return T.unit; + }), + T.provideSomeLayer(DocumentTypeMapState.Live(layer)), + )); + + // TODO: deconstruct environment state + // const run = createPool(); + // const environmentSeed = {}; + + const { dataErrors, documents } = yield* $( + pipe( + allRelativeFilePaths, + // TODO: parallalize + T.forEachParN(concurrencyLimit, (relativeFilePath) => + makeCacheItemFromFilePath({ + relativeFilePath, + filePathPatternMap, + coreSchemaDef, + contentDirPath, + options, + previousCache, + contentTypeMap, + }), + ), + T.map(Chunk.partitionThese), + T.map(({ tuple: [errors, docs] }) => ({ dataErrors: Chunk.toArray(errors), documents: Chunk.toArray(docs) })), + ), + ) + + const singletonDataErrors = yield* $(validateSingletonDocuments({ coreSchemaDef, filePathPatternMap })) + + yield* $( + FetchDataError.handleErrors({ + errors: [...dataErrors, ...singletonDataErrors], + documentCount: allRelativeFilePaths.length, + flags, + options, + schemaDef: coreSchemaDef, + contentDirPath, + verbose, + }), + ) + + const cacheItemsMap = Object.fromEntries(documents.map((_) => [_.document._id, _])) + + return { cacheItemsMap } }), - ) - - const cacheItemsMap = Object.fromEntries(documents.map((_) => [_.document._id, _])) - - return { cacheItemsMap } - }), - provideDocumentTypeMapState, - OT.withSpan('@contentlayer/source-local/fetchData:fetchAllDocuments', { attributes: { contentDirPath } }), - ) + // TODO: Figure out life cycle of this. + provideDocumentTypeMapState, + OT.withSpan('@contentlayer/source-local/fetchData:fetchAllDocuments', { attributes: { contentDirPath } }), + ) const getAllRelativeFilePaths = ({ - contentDirPath, - contentDirInclude, - contentDirExclude, + contentDirPath, + contentDirInclude, + contentDirExclude, }: { - contentDirPath: string - contentDirInclude: readonly PosixFilePath[] - contentDirExclude: readonly PosixFilePath[] -}): T.Effect => { - const getPatternPrefix = (paths_: readonly string[]) => { - const paths = paths_ - .map((_) => _.trim()) - .filter((_) => _ !== '.' && _ !== './') - .map((_) => (_.endsWith('/') ? _ : `${_}/`)) - - if (paths.length === 0) return '' - if (paths.length === 1) return paths[0] - return `{${paths.join(',')}}` - } - - const filePathPattern = '**/*.{md,mdx,json,yaml,yml}' - const pattern = `${getPatternPrefix(contentDirInclude)}${filePathPattern}` - - return pipe( - T.tryCatchPromise( - () => glob(pattern, { cwd: contentDirPath, ignore: asMutableArray(contentDirExclude) }), - (error) => new fs.UnknownFSError({ error }), - ), - T.map((_) => _.map(posixFilePath)), - OT.withSpan('@contentlayer/source-local/fetchData:getAllRelativeFilePaths'), - ) + contentDirPath: string + contentDirInclude: readonly PosixFilePath[] + contentDirExclude: readonly PosixFilePath[] + }): T.Effect => { + const getPatternPrefix = (paths_: readonly string[]) => { + const paths = paths_ + .map((_) => _.trim()) + .filter((_) => _ !== '.' && _ !== './') + .map((_) => (_.endsWith('/') ? _ : `${_}/`)) + + if (paths.length === 0) return '' + if (paths.length === 1) return paths[0] + return `{${paths.join(',')}}` + } + + const filePathPattern = '**/*.{md,mdx,json,yaml,yml}' + const pattern = `${getPatternPrefix(contentDirInclude)}${filePathPattern}` + + return pipe( + T.tryCatchPromise( + () => glob(pattern, { cwd: contentDirPath, ignore: asMutableArray(contentDirExclude) }), + (error) => new fs.UnknownFSError({ error }), + ), + T.map((_) => _.map(posixFilePath)), + OT.withSpan('@contentlayer/source-local/fetchData:getAllRelativeFilePaths'), + ) } const validateSingletonDocuments = ({ - coreSchemaDef, - filePathPatternMap, + coreSchemaDef, + filePathPatternMap, }: { - coreSchemaDef: core.SchemaDef - filePathPatternMap: FilePathPatternMap -}): T.Effect => - T.gen(function* ($) { - const singletonDocumentDefs = Object.values(coreSchemaDef.documentTypeDefMap).filter( - (documentTypeDef) => documentTypeDef.isSingleton, - ) - - const documentTypeMap = yield* $(DocumentTypeMapState.get) - - const invertedFilePathPattnernMap = invertRecord(filePathPatternMap) - - return singletonDocumentDefs - .filter( - (documentTypeDef) => - pipe( - documentTypeMap.getFilePaths(documentTypeDef.name), - O.map((_) => _.length), - O.getOrElse(() => 0), - ) !== 1, - ) - .map( - (documentTypeDef) => - new FetchDataError.SingletonDocumentNotFoundError({ - documentTypeName: documentTypeDef.name, - filePath: invertedFilePathPattnernMap[documentTypeDef.name], - }), - ) - }) + coreSchemaDef: core.SchemaDef + filePathPatternMap: FilePathPatternMap + }): T.Effect => + T.gen(function* ($) { + const singletonDocumentDefs = Object.values(coreSchemaDef.documentTypeDefMap).filter( + (documentTypeDef) => documentTypeDef.isSingleton, + ) + + const documentTypeMap = yield* $(DocumentTypeMapState.get) + + const invertedFilePathPattnernMap = invertRecord(filePathPatternMap) + + return singletonDocumentDefs + .filter( + (documentTypeDef) => + pipe( + documentTypeMap.getFilePaths(documentTypeDef.name), + O.map((_) => _.length), + O.getOrElse(() => 0), + ) !== 1, + ) + .map( + (documentTypeDef) => + new FetchDataError.SingletonDocumentNotFoundError({ + documentTypeName: documentTypeDef.name, + filePath: invertedFilePathPattnernMap[documentTypeDef.name], + }), + ) + }) const invertRecord = (record: Record): Record => - pipe(Object.entries(record), (entries) => entries.map(([key, value]) => [value, key]), Object.fromEntries) + pipe(Object.entries(record), (entries) => entries.map(([key, value]) => [value, key]), Object.fromEntries) diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index e1dcec0b..2b26703c 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -19,11 +19,13 @@ export type Input = { contentTypeMap: ContentTypeMap } +// FIXME: naming export type DTO = { input: Input; environmentSeed: any; } +// FIXME: naming // This runs on the host, what is passed into the worker at `pool.run` has to // be serializable. export function createPool() { @@ -34,10 +36,10 @@ export function createPool() { }); return (dto: DTO): T.Effect< - OT.HasTracer & HasConsole & HasDocumentTypeMapState, - never, - These.These -> => + OT.HasTracer & HasConsole & HasDocumentTypeMapState, + never, + These.These + > => pipe( T.promise(() => pool.run(dto, {name: 'makeCacheItemFromFilePath'})), T.chain(({_tag, value}) => @@ -55,7 +57,7 @@ export function createPool() { // the return value has to be serializable. export const makeCacheItemFromFilePath = (dto: DTO) => { // TODO: construct env - const env: OT.HasTracer & HasConsole & HasDocumentTypeMapState = undefined; + const env = undefined as unknown as OT.HasTracer & HasConsole & HasDocumentTypeMapState; return pipe( _.makeCacheItemFromFilePath(dto.input), T.fold( From 266d2679f36c04568748a330897a283576c655da Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 13:20:34 +0200 Subject: [PATCH 05/13] build: add eslint prettier plugin --- .editorconfig | 11 +++++++++++ .eslintrc | 3 ++- package.json | 1 + yarn.lock | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..9ab39bbc --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true + +[*.{js,jsx,ts,tsx,json,mjs}] +charset = utf-8 +indent_style = space +trim_trailing_whitespace = true +indent_size = 2 diff --git a/.eslintrc b/.eslintrc index e8dc0da9..8d4a2b73 100644 --- a/.eslintrc +++ b/.eslintrc @@ -6,9 +6,10 @@ }, "ignorePatterns": ["packages/_archive/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"], "parser": "@typescript-eslint/parser", - "plugins": ["@typescript-eslint", "simple-import-sort", "import"], + "plugins": ["prettier", "@typescript-eslint", "simple-import-sort", "import"], "extends": ["plugin:react-hooks/recommended", "plugin:@typescript-eslint/recommended", "prettier"], "rules": { + "prettier/prettier": "error", "simple-import-sort/imports": "error", "import/no-duplicates": "warn", // "func-style": ["warn", "expression"], diff --git a/package.json b/package.json index 334815b9..ac3e9814 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "eslint": "^8.15.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-import": "^2.26.0", + "eslint-plugin-prettier": "^4.2.1", "eslint-plugin-react-hooks": "^4.5.0", "eslint-plugin-simple-import-sort": "^7.0.0", "prettier": "^2.6.2", diff --git a/yarn.lock b/yarn.lock index 2c694ebe..fc7dd98a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6615,6 +6615,7 @@ __metadata: eslint: ^8.15.0 eslint-config-prettier: ^8.5.0 eslint-plugin-import: ^2.26.0 + eslint-plugin-prettier: ^4.2.1 eslint-plugin-react-hooks: ^4.5.0 eslint-plugin-simple-import-sort: ^7.0.0 prettier: ^2.6.2 @@ -8530,6 +8531,21 @@ __metadata: languageName: node linkType: hard +"eslint-plugin-prettier@npm:^4.2.1": + version: 4.2.1 + resolution: "eslint-plugin-prettier@npm:4.2.1" + dependencies: + prettier-linter-helpers: ^1.0.0 + peerDependencies: + eslint: ">=7.28.0" + prettier: ">=2.0.0" + peerDependenciesMeta: + eslint-config-prettier: + optional: true + checksum: b9e839d2334ad8ec7a5589c5cb0f219bded260839a857d7a486997f9870e95106aa59b8756ff3f37202085ebab658de382b0267cae44c3a7f0eb0bcc03a4f6d6 + languageName: node + linkType: hard + "eslint-plugin-react-hooks@npm:^4.2.0": version: 4.6.0 resolution: "eslint-plugin-react-hooks@npm:4.6.0" @@ -9247,6 +9263,13 @@ __metadata: languageName: node linkType: hard +"fast-diff@npm:^1.1.2": + version: 1.2.0 + resolution: "fast-diff@npm:1.2.0" + checksum: 1b5306eaa9e826564d9e5ffcd6ebd881eb5f770b3f977fcbf38f05c824e42172b53c79920e8429c54eb742ce15a0caf268b0fdd5b38f6de52234c4a8368131ae + languageName: node + linkType: hard + "fast-glob@npm:^3.0.3, fast-glob@npm:^3.1.1": version: 3.2.7 resolution: "fast-glob@npm:3.2.7" @@ -16321,6 +16344,15 @@ fsevents@~2.3.2: languageName: node linkType: hard +"prettier-linter-helpers@npm:^1.0.0": + version: 1.0.0 + resolution: "prettier-linter-helpers@npm:1.0.0" + dependencies: + fast-diff: ^1.1.2 + checksum: 00ce8011cf6430158d27f9c92cfea0a7699405633f7f1d4a45f07e21bf78e99895911cbcdc3853db3a824201a7c745bd49bfea8abd5fb9883e765a90f74f8392 + languageName: node + linkType: hard + "prettier@npm:^1.19.1": version: 1.19.1 resolution: "prettier@npm:1.19.1" From d814ae1a4f6a7f5dc1a9e6896a12a4b8c6a93b4d Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 13:25:29 +0200 Subject: [PATCH 06/13] feat(@contentlayer): make documentTypeMap explicit in makeCacheItemFromFilePath --- .../src/__test__/fetchData/utils.ts | 4 + .../src/fetchData/DocumentTypeMap.ts | 51 +-- .../src/fetchData/fetchAllDocuments.ts | 354 ++++++++---------- .../source-files/src/fetchData/index.ts | 4 +- .../fetchData/makeCacheItemFromFilePath.ts | 47 ++- .../makeCacheItemFromFilePath.worker.ts | 142 ++++--- .../src/fetchData/validateDocumentData.ts | 61 +-- 7 files changed, 366 insertions(+), 297 deletions(-) diff --git a/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts b/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts index ef5530e5..eaa3bb82 100644 --- a/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts +++ b/packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts @@ -54,6 +54,10 @@ export const runTest = async ({ previousCache: undefined, contentTypeMap, }), + // TODO: DocumentTypeMap is now explicitly returned from + // makeCacheItemFromFilePath in [1]. Verify that it is tested somewhere + // else, as we're throwing it away here. + T.map((_) => _.tuple[0]), These.effectToEither, ), ) diff --git a/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts b/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts index fc82f916..b98833f9 100644 --- a/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts +++ b/packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts @@ -1,46 +1,47 @@ import type { PosixFilePath } from '@contentlayer/utils' -import type { Has } from '@contentlayer/utils/effect' -import { HashMap, O, pipe, State, T, Tagged } from '@contentlayer/utils/effect' +import type { Has, Option, These } from '@contentlayer/utils/effect' +import { HashMap, O, pipe, State, T, Tagged, Tuple } from '@contentlayer/utils/effect' export type DocumentTypeName = string export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{ - readonly map: HashMap.HashMap + readonly map: HashMap.HashMap }> { - static init = () => new DocumentTypeMap({ map: HashMap.make() }) + static init = () => new DocumentTypeMap({ map: HashMap.make() }) - add = (documentTypeName: DocumentTypeName, filePath: PosixFilePath) => { - const oldPaths = pipe( - HashMap.get_(this.map, documentTypeName), - O.getOrElse(() => []), - ) + add = (documentTypeName: DocumentTypeName, filePath: PosixFilePath) => { + const oldPaths = pipe( + HashMap.get_(this.map, documentTypeName), + O.getOrElse(() => []), + ) - return new DocumentTypeMap({ - map: HashMap.set_(this.map, documentTypeName, [...oldPaths, filePath]), - }) - } + return new DocumentTypeMap({ + map: HashMap.set_(this.map, documentTypeName, [...oldPaths, filePath]), + }) + } - getFilePaths = (documentTypeName: DocumentTypeName): O.Option => - HashMap.get_(this.map, documentTypeName) + getFilePaths = (documentTypeName: DocumentTypeName): O.Option => + HashMap.get_(this.map, documentTypeName) } /** -* This state is needed for certain kinds of error handling (e.g. to check if singleton documents have been provided) -*/ + * This state is needed for certain kinds of error handling (e.g. to check if singleton documents have been provided) + */ export const DocumentTypeMapState = State.State(DocumentTypeMap._tag) -export const serialize = () => pipe( +export const serialize = () => + pipe( DocumentTypeMapState.get, // FIXME: unsafe - T.chain(map => T.succeedWith(() => JSON.stringify(map))), -); + T.chain((map) => T.succeedWith(() => JSON.stringify(map))), + ) export function provideFromSerialized(serialized: string) { - return pipe( - // FIXME: unsafe - T.succeedWith(() => JSON.parse(serialized)), - T.map(map => T.provideSomeLayer(DocumentTypeMapState.Live(new DocumentTypeMap(map)))) - ) + return pipe( + // FIXME: unsafe + T.succeedWith(() => JSON.parse(serialized)), + T.map((map) => T.provideSomeLayer(DocumentTypeMapState.Live(new DocumentTypeMap(map)))), + ) } export const provideDocumentTypeMapState = T.provideSomeLayer(DocumentTypeMapState.Live(DocumentTypeMap.init())) diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index c7eb6cf5..22959d5b 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -1,218 +1,186 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' import { asMutableArray, posixFilePath } from '@contentlayer/utils' -import type { HasConsole} from '@contentlayer/utils/effect'; -import { Chunk, HashMap , O, OT, pipe, T } from '@contentlayer/utils/effect' +import type { HasConsole } from '@contentlayer/utils/effect' +import { Chunk, E, HashMap, O, Option, OT, pipe, T, These, Tp } from '@contentlayer/utils/effect' import { fs } from '@contentlayer/utils/node' -import { Has } from '@effect-ts/core'; import glob from 'fast-glob' import * as os from 'node:os' import { FetchDataError } from '../errors/index.js' import type { Flags } from '../index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' -import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' -import { DocumentTypeMapState, provideDocumentTypeMapState } from './DocumentTypeMap.js' -import * as SF from './DocumentTypeMap.js' +import { DocumentTypeMap, provideDocumentTypeMapState } from './DocumentTypeMap.js' import { makeCacheItemFromFilePath } from './makeCacheItemFromFilePath.js' // import {createPool} from './makeCacheItemFromFilePath.worker.js' export const fetchAllDocuments = ({ - coreSchemaDef, - filePathPatternMap, - contentDirPath, - contentDirInclude, - contentDirExclude, - contentTypeMap, - flags, - options, - previousCache, - verbose, + coreSchemaDef, + filePathPatternMap, + contentDirPath, + contentDirInclude, + contentDirExclude, + contentTypeMap, + flags, + options, + previousCache, + verbose, }: { - coreSchemaDef: core.SchemaDef - filePathPatternMap: FilePathPatternMap - contentDirPath: PosixFilePath - contentDirInclude: readonly PosixFilePath[] - contentDirExclude: readonly PosixFilePath[] - contentTypeMap: ContentTypeMap - flags: Flags - options: core.PluginOptions - previousCache: core.DataCache.Cache | undefined - verbose: boolean - }): T.Effect => - pipe( - T.gen(function* ($) { - const allRelativeFilePaths = yield* $( - getAllRelativeFilePaths({ contentDirPath, contentDirInclude, contentDirExclude }), - ) - - const concurrencyLimit = os.cpus().length - - // host -> worker - const serialized = yield* $(pipe( - SF.DocumentTypeMapState.update(_ => _.add('foo', posixFilePath('bar'))), - T.chain(() => DocumentTypeMapState.get), - T.tap(map => { - console.log('start', map); - return T.unit; - }), - T.chain(map => T.succeedWith(() => { - const r: Record = {}; - for (const [k, v] of map.map) { - r[k] = v; - } - return JSON.stringify(r); - })), - T.tap(serialized => { - console.log('serialized', serialized); - return T.unit; - }), - SF.provideDocumentTypeMapState, - )); - - // worker input - const layer = yield* $(pipe( - T.succeedWith(() => { - const i: Record = JSON.parse(serialized); - return Object.entries(i).reduce((map, [key, value]) => { - return HashMap.set(key, value)(map) - }, HashMap.make()); - }), - T.tap(deserialized => { - console.log('deserialized', deserialized); - return T.unit; - }), - T.map(map => new SF.DocumentTypeMap({map})), - T.tap(instantiated => { - console.log('instantiated', instantiated); - return T.unit; - }), - )); - - // worker -> host - yield* $(pipe( - SF.DocumentTypeMapState.update(_ => _.add('qux', posixFilePath('baz'))), - T.chain(() => SF.DocumentTypeMapState.get), - T.tap(end => { - console.log('end', end); - return T.unit; - }), - T.provideSomeLayer(DocumentTypeMapState.Live(layer)), - )); - - // TODO: deconstruct environment state - // const run = createPool(); - // const environmentSeed = {}; - - const { dataErrors, documents } = yield* $( - pipe( - allRelativeFilePaths, - // TODO: parallalize - T.forEachParN(concurrencyLimit, (relativeFilePath) => - makeCacheItemFromFilePath({ - relativeFilePath, - filePathPatternMap, - coreSchemaDef, - contentDirPath, - options, - previousCache, - contentTypeMap, - }), - ), - T.map(Chunk.partitionThese), - T.map(({ tuple: [errors, docs] }) => ({ dataErrors: Chunk.toArray(errors), documents: Chunk.toArray(docs) })), - ), - ) - - const singletonDataErrors = yield* $(validateSingletonDocuments({ coreSchemaDef, filePathPatternMap })) - - yield* $( - FetchDataError.handleErrors({ - errors: [...dataErrors, ...singletonDataErrors], - documentCount: allRelativeFilePaths.length, - flags, - options, - schemaDef: coreSchemaDef, - contentDirPath, - verbose, - }), - ) - - const cacheItemsMap = Object.fromEntries(documents.map((_) => [_.document._id, _])) - - return { cacheItemsMap } + coreSchemaDef: core.SchemaDef + filePathPatternMap: FilePathPatternMap + contentDirPath: PosixFilePath + contentDirInclude: readonly PosixFilePath[] + contentDirExclude: readonly PosixFilePath[] + contentTypeMap: ContentTypeMap + flags: Flags + options: core.PluginOptions + previousCache: core.DataCache.Cache | undefined + verbose: boolean +}): T.Effect => + pipe( + T.gen(function* ($) { + const allRelativeFilePaths = yield* $( + getAllRelativeFilePaths({ contentDirPath, contentDirInclude, contentDirExclude }), + ) + + const concurrencyLimit = os.cpus().length + + // TODO createPool + const { dataErrors, documents, documentTypeMap } = yield* $( + pipe( + allRelativeFilePaths, + // TODO: parallalize + T.forEachParN(concurrencyLimit, (relativeFilePath) => + makeCacheItemFromFilePath({ + relativeFilePath, + filePathPatternMap, + coreSchemaDef, + contentDirPath, + options, + previousCache, + contentTypeMap, + }), + ), + T.map((chunk) => { + let errors = Chunk.empty() + let values = Chunk.empty() + let documentTypeMap = DocumentTypeMap.init() + + Chunk.forEach_(chunk, (_) => { + const [a, documentTypeTupleOption] = _.tuple + if (Option.isSome(documentTypeTupleOption)) { + const [documentTypeName, relativeFilePath] = documentTypeTupleOption.value.tuple + documentTypeMap = documentTypeMap.add(documentTypeName, relativeFilePath) + } + const res = These.result(a) + if (E.isLeft(res)) { + errors = Chunk.append_(errors, res.left) + } else { + values = Chunk.append_(values, res.right.tuple[0]) + const warning = res.right.tuple[1] + if (O.isSome(warning)) { + errors = Chunk.append_(errors, warning.value) + } + } + }) + + return Tp.tuple(errors, values, documentTypeMap) + }), + T.map(({ tuple: [errors, docs, documentTypeMap] }) => ({ + dataErrors: Chunk.toArray(errors), + documents: Chunk.toArray(docs), + documentTypeMap, + })), + ), + ) + + const singletonDataErrors = validateSingletonDocuments({ coreSchemaDef, filePathPatternMap, documentTypeMap }) + + yield* $( + FetchDataError.handleErrors({ + errors: [...dataErrors, ...singletonDataErrors], + documentCount: allRelativeFilePaths.length, + flags, + options, + schemaDef: coreSchemaDef, + contentDirPath, + verbose, }), - // TODO: Figure out life cycle of this. - provideDocumentTypeMapState, - OT.withSpan('@contentlayer/source-local/fetchData:fetchAllDocuments', { attributes: { contentDirPath } }), - ) + ) + + const cacheItemsMap = Object.fromEntries(documents.map((_) => [_.document._id, _])) + + return { cacheItemsMap } + }), + provideDocumentTypeMapState, + OT.withSpan('@contentlayer/source-local/fetchData:fetchAllDocuments', { attributes: { contentDirPath } }), + ) const getAllRelativeFilePaths = ({ - contentDirPath, - contentDirInclude, - contentDirExclude, + contentDirPath, + contentDirInclude, + contentDirExclude, }: { - contentDirPath: string - contentDirInclude: readonly PosixFilePath[] - contentDirExclude: readonly PosixFilePath[] - }): T.Effect => { - const getPatternPrefix = (paths_: readonly string[]) => { - const paths = paths_ - .map((_) => _.trim()) - .filter((_) => _ !== '.' && _ !== './') - .map((_) => (_.endsWith('/') ? _ : `${_}/`)) - - if (paths.length === 0) return '' - if (paths.length === 1) return paths[0] - return `{${paths.join(',')}}` - } - - const filePathPattern = '**/*.{md,mdx,json,yaml,yml}' - const pattern = `${getPatternPrefix(contentDirInclude)}${filePathPattern}` - - return pipe( - T.tryCatchPromise( - () => glob(pattern, { cwd: contentDirPath, ignore: asMutableArray(contentDirExclude) }), - (error) => new fs.UnknownFSError({ error }), - ), - T.map((_) => _.map(posixFilePath)), - OT.withSpan('@contentlayer/source-local/fetchData:getAllRelativeFilePaths'), - ) + contentDirPath: string + contentDirInclude: readonly PosixFilePath[] + contentDirExclude: readonly PosixFilePath[] +}): T.Effect => { + const getPatternPrefix = (paths_: readonly string[]) => { + const paths = paths_ + .map((_) => _.trim()) + .filter((_) => _ !== '.' && _ !== './') + .map((_) => (_.endsWith('/') ? _ : `${_}/`)) + + if (paths.length === 0) return '' + if (paths.length === 1) return paths[0] + return `{${paths.join(',')}}` + } + + const filePathPattern = '**/*.{md,mdx,json,yaml,yml}' + const pattern = `${getPatternPrefix(contentDirInclude)}${filePathPattern}` + + return pipe( + T.tryCatchPromise( + () => glob(pattern, { cwd: contentDirPath, ignore: asMutableArray(contentDirExclude) }), + (error) => new fs.UnknownFSError({ error }), + ), + T.map((_) => _.map(posixFilePath)), + OT.withSpan('@contentlayer/source-local/fetchData:getAllRelativeFilePaths'), + ) } const validateSingletonDocuments = ({ - coreSchemaDef, - filePathPatternMap, + coreSchemaDef, + filePathPatternMap, + documentTypeMap, }: { - coreSchemaDef: core.SchemaDef - filePathPatternMap: FilePathPatternMap - }): T.Effect => - T.gen(function* ($) { - const singletonDocumentDefs = Object.values(coreSchemaDef.documentTypeDefMap).filter( - (documentTypeDef) => documentTypeDef.isSingleton, - ) - - const documentTypeMap = yield* $(DocumentTypeMapState.get) - - const invertedFilePathPattnernMap = invertRecord(filePathPatternMap) - - return singletonDocumentDefs - .filter( - (documentTypeDef) => - pipe( - documentTypeMap.getFilePaths(documentTypeDef.name), - O.map((_) => _.length), - O.getOrElse(() => 0), - ) !== 1, - ) - .map( - (documentTypeDef) => - new FetchDataError.SingletonDocumentNotFoundError({ - documentTypeName: documentTypeDef.name, - filePath: invertedFilePathPattnernMap[documentTypeDef.name], - }), - ) - }) + coreSchemaDef: core.SchemaDef + filePathPatternMap: FilePathPatternMap + documentTypeMap: DocumentTypeMap +}): FetchDataError.SingletonDocumentNotFoundError[] => { + const singletonDocumentDefs = Object.values(coreSchemaDef.documentTypeDefMap).filter( + (documentTypeDef) => documentTypeDef.isSingleton, + ) + + const invertedFilePathPattnernMap = invertRecord(filePathPatternMap) + + return singletonDocumentDefs + .filter( + (documentTypeDef) => + pipe( + documentTypeMap.getFilePaths(documentTypeDef.name), + O.map((_) => _.length), + O.getOrElse(() => 0), + ) !== 1, + ) + .map( + (documentTypeDef) => + new FetchDataError.SingletonDocumentNotFoundError({ + documentTypeName: documentTypeDef.name, + filePath: invertedFilePathPattnernMap[documentTypeDef.name], + }), + ) +} const invertRecord = (record: Record): Record => - pipe(Object.entries(record), (entries) => entries.map(([key, value]) => [value, key]), Object.fromEntries) + pipe(Object.entries(record), (entries) => entries.map(([key, value]) => [value, key]), Object.fromEntries) diff --git a/packages/@contentlayer/source-files/src/fetchData/index.ts b/packages/@contentlayer/source-files/src/fetchData/index.ts index f655cf8a..a557896b 100644 --- a/packages/@contentlayer/source-files/src/fetchData/index.ts +++ b/packages/@contentlayer/source-files/src/fetchData/index.ts @@ -163,8 +163,8 @@ const updateCacheEntry = ({ previousCache: cache, contentTypeMap, }), - // NOTE in this code path the DocumentTypeMapState is not used - provideDocumentTypeMapState, + // We're not interested in the documentTypeMap in this branch, so we discard it. + T.map((_) => _.tuple[0]), These.effectTapSuccess((cacheItem) => T.succeedWith(() => { cache.cacheItemsMap[event.relativeFilePath] = cacheItem diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts index dc82fcfe..fbe49ce7 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts @@ -2,7 +2,7 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' import { filePathJoin } from '@contentlayer/utils' import type { HasConsole } from '@contentlayer/utils/effect' -import { identity, O, OT, pipe, T, These } from '@contentlayer/utils/effect' +import { E, identity, O, Option, OT, pipe, T, These, Tuple } from '@contentlayer/utils/effect' import { fs } from '@contentlayer/utils/node' import matter from 'gray-matter' import yaml from 'yaml' @@ -10,7 +10,7 @@ import yaml from 'yaml' import { FetchDataError } from '../errors/index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import { makeAndProvideDocumentContext } from './DocumentContext.js' -import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' +import type { DocumentTypeName, HasDocumentTypeMapState } from './DocumentTypeMap.js' import { DocumentTypeMapState } from './DocumentTypeMap.js' import { makeDocument } from './mapping.js' import type { RawContent, RawContentJSON, RawContentMarkdown, RawContentMDX, RawContentYAML } from './types.js' @@ -33,9 +33,14 @@ export const makeCacheItemFromFilePath = ({ previousCache: core.DataCache.Cache | undefined contentTypeMap: ContentTypeMap }): T.Effect< - OT.HasTracer & HasConsole & HasDocumentTypeMapState, + OT.HasTracer & HasConsole, never, - These.These + Tuple.Tuple< + [ + These.These, + Option.Option>, + ] + > > => pipe( T.gen(function* ($) { @@ -56,14 +61,17 @@ export const makeCacheItemFromFilePath = ({ previousCache.cacheItemsMap[relativeFilePath]!.hasWarnings === false ) { const cacheItem = previousCache.cacheItemsMap[relativeFilePath]! - yield* $(DocumentTypeMapState.update((_) => _.add(cacheItem.documentTypeName, relativeFilePath))) + const documentTypeTuple = Option.some(Tuple.tuple(cacheItem.documentTypeName, relativeFilePath)) - return These.succeed(cacheItem) + return Tuple.tuple(These.succeed(cacheItem), documentTypeTuple) } const rawContent = yield* $(processRawContent({ fullFilePath, relativeFilePath })) - const [{ documentTypeDef }, warnings] = yield* $( + const { + data: [{ documentTypeDef }, warnings], + documentTypeTuple, + } = yield* $( pipe( validateDocumentData({ rawContent, @@ -74,8 +82,14 @@ export const makeCacheItemFromFilePath = ({ contentDirPath, contentTypeMap, }), - T.chain(These.toEffect), T.map((_) => _.tuple), + T.chain(([data, documentTypeTuple]) => + pipe( + These.toEffect(data), + T.map((_) => _.tuple), + T.map((data) => ({ data, documentTypeTuple })), + ), + ), ), ) @@ -102,9 +116,17 @@ export const makeCacheItemFromFilePath = ({ }) } - return These.warnOption( - { document, documentHash, hasWarnings: O.isSome(warnings), documentTypeName: documentTypeDef.name }, - warnings, + return Tuple.tuple( + These.warnOption( + { + document, + documentHash, + hasWarnings: O.isSome(warnings), + documentTypeName: documentTypeDef.name, + }, + warnings, + ), + documentTypeTuple, ) }), OT.withSpan('@contentlayer/source-local/fetchData:makeCacheItemFromFilePath'), @@ -118,7 +140,8 @@ export const makeCacheItemFromFilePath = ({ return error } }), - These.effectThese, + T.either, + T.map(E.fold((e) => Tuple.tuple(These.fail(e), Option.none), identity)), ) const processRawContent = ({ diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index 2b26703c..51a383eb 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,12 +1,14 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' -import type { HasConsole, OT , These } from '@contentlayer/utils/effect'; -import {pipe, T } from '@contentlayer/utils/effect' -import Pool from 'piscina'; +import { provideDummyTracing } from '@contentlayer/utils' +import type { HasConsole, OT, These } from '@contentlayer/utils/effect' +import { pipe, provideConsole, T } from '@contentlayer/utils/effect' +import Pool from 'piscina' import type { FetchDataError } from '../errors/index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' +import { provideDocumentTypeMapState } from './DocumentTypeMap.js' import * as _ from './makeCacheItemFromFilePath.js' export type Input = { @@ -19,52 +21,108 @@ export type Input = { contentTypeMap: ContentTypeMap } -// FIXME: naming -export type DTO = { - input: Input; - environmentSeed: any; -} - // FIXME: naming // This runs on the host, what is passed into the worker at `pool.run` has to // be serializable. export function createPool() { - // I believe, by default, #workers = #cpu cores, which is probably what we want? - const pool = new Pool({ - // FIXME: get path dynamically - filename: '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', - }); + // I believe, by default, #workers = #cpu cores, which is probably what we want? + const pool = new Pool({ + // FIXME: get path dynamically + filename: + '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', + }) - return (dto: DTO): T.Effect< - OT.HasTracer & HasConsole & HasDocumentTypeMapState, - never, - These.These - > => - pipe( - T.promise(() => pool.run(dto, {name: 'makeCacheItemFromFilePath'})), - T.chain(({_tag, value}) => - T.if_( - _tag === 'right', - () => T.succeed(value), - // FIXME: Signature claims it doesn't fail. - () => T.die(value), - ), - ), - ) + return ( + payload: Input, + ): T.Effect< + OT.HasTracer & HasConsole & HasDocumentTypeMapState, + never, + These.These + > => + pipe( + T.promise(() => pool.run(payload, { name: 'makeCacheItemFromFilePath' })), + T.chain(({ _tag, value }) => + T.if_( + _tag === 'right', + () => T.succeed(value), + // FIXME: Signature claims it doesn't fail. + () => T.die(value), + ), + ), + ) } // This runs in the worker, with the input coming via the "wire" from the host, // the return value has to be serializable. -export const makeCacheItemFromFilePath = (dto: DTO) => { - // TODO: construct env - const env = undefined as unknown as OT.HasTracer & HasConsole & HasDocumentTypeMapState; - return pipe( - _.makeCacheItemFromFilePath(dto.input), - T.fold( - value => ({_tag: 'left', value} as const), - value => ({_tag: 'right', value} as const), - ), - T.provideAll(env), - T.runPromise, - ); +export function makeCacheItemFromFilePath(payload: Input) { + return pipe( + _.makeCacheItemFromFilePath(payload), + // T.fold( + // value => ({ _tag: 'left', value } as const), + // value => ({ _tag: 'right', value } as const) + // ), + provideDocumentTypeMapState, + provideConsole, + provideDummyTracing, + T.runPromise, + ) } + +// const serialized = yield* $( +// pipe( +// SF.DocumentTypeMapState.update((_) => _.add('foo', posixFilePath('bar'))), +// T.chain(() => DocumentTypeMapState.get), +// T.tap((map) => { +// console.log('start', map) +// return T.unit +// }), +// T.chain((map) => +// T.succeedWith(() => { +// const r: Record = {} +// for (const [k, v] of map.map) { +// r[k] = v +// } +// return JSON.stringify(r) +// }), +// ), +// T.tap((serialized) => { +// console.log('serialized', serialized) +// return T.unit +// }), +// SF.provideDocumentTypeMapState, +// ), +// ) +// +// // worker input +// const layer = yield* $( +// pipe( +// T.succeedWith(() => { +// const i: Record = JSON.parse(serialized) +// return Object.entries(i).reduce((map, [key, value]) => { +// return HashMap.set(key, value)(map) +// }, HashMap.make()) +// }), +// T.tap((deserialized) => { +// console.log('deserialized', deserialized) +// return T.unit +// }), +// T.map((map) => new SF.DocumentTypeMap({ map })), +// T.tap((instantiated) => { +// console.log('instantiated', instantiated) +// return T.unit +// }), +// ), +// ) +// +// // worker -> host +// yield* $( +// pipe( +// SF.DocumentTypeMapState.update((_) => _.add('qux', posixFilePath('baz'))), +// T.chain(() => SF.DocumentTypeMapState.get), +// T.tap((end) => { +// console.log('end', end) +// return T.unit +// }), +// T.provideSomeLayer(DocumentTypeMapState.Live(layer)), +// ), +// ) diff --git a/packages/@contentlayer/source-files/src/fetchData/validateDocumentData.ts b/packages/@contentlayer/source-files/src/fetchData/validateDocumentData.ts index 054b69bc..ce2d3270 100644 --- a/packages/@contentlayer/source-files/src/fetchData/validateDocumentData.ts +++ b/packages/@contentlayer/source-files/src/fetchData/validateDocumentData.ts @@ -1,15 +1,14 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' import { filePathJoin } from '@contentlayer/utils' -import { O, OT, pipe, T, These } from '@contentlayer/utils/effect' +import { O, Option, OT, pipe, T, These, Tuple } from '@contentlayer/utils/effect' import { fs } from '@contentlayer/utils/node' import micromatch from 'micromatch' import { FetchDataError } from '../errors/index.js' import type { DocumentContentType, FilePathPatternMap } from '../index.js' import type { ContentTypeMap } from '../types.js' -import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' -import { DocumentTypeMapState } from './DocumentTypeMap.js' +import type { DocumentTypeName } from './DocumentTypeMap.js' import type { RawContent } from './types.js' type ValidateDocumentDataError = @@ -39,9 +38,14 @@ export const validateDocumentData = ({ contentDirPath: PosixFilePath contentTypeMap: ContentTypeMap }): T.Effect< - HasDocumentTypeMapState & OT.HasTracer, + OT.HasTracer, never, - These.These + Tuple.Tuple< + [ + These.These, + Option.Option>, + ] + > > => pipe( T.gen(function* ($) { @@ -51,27 +55,36 @@ export const validateDocumentData = ({ if (documentDefName === undefined) { const typeFieldName = options.fieldOptions.typeFieldName - return These.fail( - new FetchDataError.CouldNotDetermineDocumentTypeError({ documentFilePath: relativeFilePath, typeFieldName }), + return Tuple.tuple( + These.fail( + new FetchDataError.CouldNotDetermineDocumentTypeError({ + documentFilePath: relativeFilePath, + typeFieldName, + }), + ), + Option.none, ) } const documentTypeDef = coreSchemaDef.documentTypeDefMap[documentDefName] if (documentTypeDef === undefined) { - return These.fail( - new FetchDataError.NoSuchDocumentTypeError({ - documentTypeName: documentDefName, - documentFilePath: relativeFilePath, - }), + return Tuple.tuple( + These.fail( + new FetchDataError.NoSuchDocumentTypeError({ + documentTypeName: documentDefName, + documentFilePath: relativeFilePath, + }), + ), + Option.none, ) } const contentType = contentTypeMap[documentTypeDef.name]! const mismatchError = validateContentTypeMatchesFileExtension({ contentType, relativeFilePath }) - if (mismatchError) return These.fail(mismatchError) + if (mismatchError) return Tuple.tuple(These.fail(mismatchError), Option.none) - yield* $(DocumentTypeMapState.update((_) => _.add(documentDefName, relativeFilePath))) + const meta = Option.some(Tuple.tuple(documentDefName, relativeFilePath)) const existingDataFieldKeys = Object.keys(rawContent.fields) @@ -83,12 +96,15 @@ export const validateDocumentData = ({ (fieldDef) => !existingDataFieldKeys.includes(fieldDef.name), ) if (misingRequiredFieldDefs.length > 0) { - return These.fail( - new FetchDataError.MissingRequiredFieldsError({ - documentFilePath: relativeFilePath, - documentTypeName: documentTypeDef.name, - fieldDefsWithMissingData: misingRequiredFieldDefs, - }), + return Tuple.tuple( + These.fail( + new FetchDataError.MissingRequiredFieldsError({ + documentFilePath: relativeFilePath, + documentTypeName: documentTypeDef.name, + fieldDefsWithMissingData: misingRequiredFieldDefs, + }), + ), + meta, ) } @@ -124,13 +140,12 @@ export const validateDocumentData = ({ ) if (O.isSome(fieldValidOption)) { - return These.fail(fieldValidOption.value) + return Tuple.tuple(These.fail(fieldValidOption.value), meta) } } // TODO validate nesteds - - return These.warnOption({ documentTypeDef }, warningOption) + return Tuple.tuple(These.warnOption({ documentTypeDef }, warningOption), meta) }), OT.withSpan('validateDocumentData', { attributes: { relativeFilePath } }), ) From a53243eb899e0f916291514842f6c9698ff22319 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 14:33:39 +0200 Subject: [PATCH 07/13] feat: compute makeCacheItemFromFilePath via worker pool --- .../src/fetchData/fetchAllDocuments.ts | 4 +- .../fetchData/makeCacheItemFromFilePath.ts | 1 - .../makeCacheItemFromFilePath.worker.ts | 119 ++++-------------- 3 files changed, 27 insertions(+), 97 deletions(-) diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index 22959d5b..b5fd2687 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -11,7 +11,7 @@ import { FetchDataError } from '../errors/index.js' import type { Flags } from '../index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import { DocumentTypeMap, provideDocumentTypeMapState } from './DocumentTypeMap.js' -import { makeCacheItemFromFilePath } from './makeCacheItemFromFilePath.js' +import { fromWorkerPool } from './makeCacheItemFromFilePath.worker.js' // import {createPool} from './makeCacheItemFromFilePath.worker.js' export const fetchAllDocuments = ({ @@ -45,7 +45,7 @@ export const fetchAllDocuments = ({ const concurrencyLimit = os.cpus().length - // TODO createPool + const makeCacheItemFromFilePath = fromWorkerPool() const { dataErrors, documents, documentTypeMap } = yield* $( pipe( allRelativeFilePaths, diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts index fbe49ce7..504c3c25 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts @@ -11,7 +11,6 @@ import { FetchDataError } from '../errors/index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import { makeAndProvideDocumentContext } from './DocumentContext.js' import type { DocumentTypeName, HasDocumentTypeMapState } from './DocumentTypeMap.js' -import { DocumentTypeMapState } from './DocumentTypeMap.js' import { makeDocument } from './mapping.js' import type { RawContent, RawContentJSON, RawContentMarkdown, RawContentMDX, RawContentYAML } from './types.js' import { validateDocumentData } from './validateDocumentData.js' diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index 51a383eb..6f604431 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,30 +1,22 @@ -import type * as core from '@contentlayer/core' -import type { PosixFilePath } from '@contentlayer/utils' import { provideDummyTracing } from '@contentlayer/utils' -import type { HasConsole, OT, These } from '@contentlayer/utils/effect' import { pipe, provideConsole, T } from '@contentlayer/utils/effect' +import type { _A, _E } from '@effect-ts/core/Utils' import Pool from 'piscina' -import type { FetchDataError } from '../errors/index.js' -import type { ContentTypeMap, FilePathPatternMap } from '../types.js' -import type { HasDocumentTypeMapState } from './DocumentTypeMap.js' -import { provideDocumentTypeMapState } from './DocumentTypeMap.js' import * as _ from './makeCacheItemFromFilePath.js' -export type Input = { - relativeFilePath: PosixFilePath - filePathPatternMap: FilePathPatternMap - coreSchemaDef: core.SchemaDef - contentDirPath: PosixFilePath - options: core.PluginOptions - previousCache: core.DataCache.Cache | undefined - contentTypeMap: ContentTypeMap -} +type F = typeof _.makeCacheItemFromFilePath + +export type Left = { _tag: 'left'; value: E } +export type Right = { _tag: 'right'; value: A } +export type Either = Left | Right + +type DTO = Either<_E>, _A>> // FIXME: naming // This runs on the host, what is passed into the worker at `pool.run` has to // be serializable. -export function createPool() { +export function fromWorkerPool(): F { // I believe, by default, #workers = #cpu cores, which is probably what we want? const pool = new Pool({ // FIXME: get path dynamically @@ -32,19 +24,16 @@ export function createPool() { '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', }) - return ( - payload: Input, - ): T.Effect< - OT.HasTracer & HasConsole & HasDocumentTypeMapState, - never, - These.These - > => + return (payload) => pipe( - T.promise(() => pool.run(payload, { name: 'makeCacheItemFromFilePath' })), + T.succeedWith(() => JSON.stringify(payload)), + T.chain((value) => T.promise(() => pool.run(value, { name: 'makeCacheItemFromFilePath' }))), + T.chain((value) => T.succeedWith(() => JSON.parse(value))), T.chain(({ _tag, value }) => T.if_( _tag === 'right', - () => T.succeed(value), + // FIXME: effect + () => T.succeedWith(() => value), // FIXME: Signature claims it doesn't fail. () => T.die(value), ), @@ -52,77 +41,19 @@ export function createPool() { ) } -// This runs in the worker, with the input coming via the "wire" from the host, -// the return value has to be serializable. -export function makeCacheItemFromFilePath(payload: Input) { +// This runs IN THE WORKER, with the input coming via "wire" from the host. +// Since input and output cross a process boundary, input has to be de-serialized, output has to be serialized. +export function makeCacheItemFromFilePath(payload: string): Promise { return pipe( - _.makeCacheItemFromFilePath(payload), - // T.fold( - // value => ({ _tag: 'left', value } as const), - // value => ({ _tag: 'right', value } as const) - // ), - provideDocumentTypeMapState, + T.succeedWith[0]>(() => JSON.parse(payload)), + T.chain(_.makeCacheItemFromFilePath), + T.fold( + (value) => ({ _tag: 'left', value } as const), + (value) => ({ _tag: 'right', value } as const), + ), provideConsole, provideDummyTracing, T.runPromise, + (p) => p.then((value) => JSON.stringify(value)), ) } - -// const serialized = yield* $( -// pipe( -// SF.DocumentTypeMapState.update((_) => _.add('foo', posixFilePath('bar'))), -// T.chain(() => DocumentTypeMapState.get), -// T.tap((map) => { -// console.log('start', map) -// return T.unit -// }), -// T.chain((map) => -// T.succeedWith(() => { -// const r: Record = {} -// for (const [k, v] of map.map) { -// r[k] = v -// } -// return JSON.stringify(r) -// }), -// ), -// T.tap((serialized) => { -// console.log('serialized', serialized) -// return T.unit -// }), -// SF.provideDocumentTypeMapState, -// ), -// ) -// -// // worker input -// const layer = yield* $( -// pipe( -// T.succeedWith(() => { -// const i: Record = JSON.parse(serialized) -// return Object.entries(i).reduce((map, [key, value]) => { -// return HashMap.set(key, value)(map) -// }, HashMap.make()) -// }), -// T.tap((deserialized) => { -// console.log('deserialized', deserialized) -// return T.unit -// }), -// T.map((map) => new SF.DocumentTypeMap({ map })), -// T.tap((instantiated) => { -// console.log('instantiated', instantiated) -// return T.unit -// }), -// ), -// ) -// -// // worker -> host -// yield* $( -// pipe( -// SF.DocumentTypeMapState.update((_) => _.add('qux', posixFilePath('baz'))), -// T.chain(() => SF.DocumentTypeMapState.get), -// T.tap((end) => { -// console.log('end', end) -// return T.unit -// }), -// T.provideSomeLayer(DocumentTypeMapState.Live(layer)), -// ), -// ) From 6d8d3a2f5b498e0bf4164ac77779ccb3bd0f53db Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 16:14:16 +0200 Subject: [PATCH 08/13] feat: de-serializing errors --- .../source-files/src/errors/index.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/packages/@contentlayer/source-files/src/errors/index.ts b/packages/@contentlayer/source-files/src/errors/index.ts index 3507fe82..3e095e91 100644 --- a/packages/@contentlayer/source-files/src/errors/index.ts +++ b/packages/@contentlayer/source-files/src/errors/index.ts @@ -355,6 +355,47 @@ This is possibly a bug in Contentlayer. Please open an issue.` renderLine = () => `"${this.documentFilePath}": ${errorToString(this.error)}` } + + const tagToError = { + InvalidFrontmatterError: InvalidFrontmatterError, + InvalidMarkdownFileError: InvalidMarkdownFileError, + InvalidYamlFileError: InvalidYamlFileError, + InvalidJsonFileError: InvalidJsonFileError, + ComputedValueError: ComputedValueError, + UnsupportedFileExtension: UnsupportedFileExtension, + FileExtensionMismatch: FileExtensionMismatch, + NoSuchDocumentTypeError: NoSuchDocumentTypeError, + NoSuchNestedDocumentTypeError: NoSuchNestedDocumentTypeError, + CouldNotDetermineDocumentTypeError: CouldNotDetermineDocumentTypeError, + MissingRequiredFieldsError: MissingRequiredFieldsError, + ExtraFieldDataError: ExtraFieldDataError, + ReferencedFileDoesNotExistError: ReferencedFileDoesNotExistError, + IncompatibleFieldDataError: IncompatibleFieldDataError, + SingletonDocumentNotFoundError: SingletonDocumentNotFoundError, + UnexpectedError: UnexpectedError, + } + + export const fromSerialized = ({ + _tag, + ...rest + }: { + _tag: keyof typeof tagToError + // FIXME: Obviously, this type is a lie, arguments would be a union of + // class params, instead of an intersection. + readonly error: unknown + readonly documentFilePath: PosixFilePath + readonly documentTypeName: string + readonly fieldName: string + readonly extension: string + readonly filePath: string + readonly typeFieldName: string + readonly contentType: DocumentContentType + readonly referencedFilePath: PosixFilePath + readonly validNestedTypeNames: readonly string[] + readonly fieldDefsWithMissingData: core.FieldDef[] + readonly extraFieldEntries: readonly (readonly [fieldKey: string, fieldValue: any])[] + readonly incompatibleFieldData: readonly (readonly [fieldKey: string, fieldValue: any])[] + }): FetchDataError => new tagToError[_tag](rest) } export type SchemaError = DuplicateBodyFieldError From d8784a65869d5c4c164c197bf13c038a4d0329f5 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 16:17:09 +0200 Subject: [PATCH 09/13] fix: node worker (supposedly) OOM This fixes an issue where the piscina worker pool crashes with `FATAL ERROR: v8::FromJust Maybe value is Nothing` Supposedly, this is the worker running out of memory? No idea. As a fix, we explicitly pick boundaries for queue and thread size in piscina. --- .../src/fetchData/fetchAllDocuments.ts | 3 ++- .../makeCacheItemFromFilePath.worker.ts | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index b5fd2687..b50d6695 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -74,7 +74,8 @@ export const fetchAllDocuments = ({ } const res = These.result(a) if (E.isLeft(res)) { - errors = Chunk.append_(errors, res.left) + // FIXME: type + errors = Chunk.append_(errors, FetchDataError.fromSerialized(res.left as unknown as any)) } else { values = Chunk.append_(values, res.right.tuple[0]) const warning = res.right.tuple[1] diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index 6f604431..86a9653e 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,6 +1,8 @@ import { provideDummyTracing } from '@contentlayer/utils' import { pipe, provideConsole, T } from '@contentlayer/utils/effect' import type { _A, _E } from '@effect-ts/core/Utils' +import fs from 'node:fs' +import * as os from 'node:os' import Pool from 'piscina' import * as _ from './makeCacheItemFromFilePath.js' @@ -14,11 +16,16 @@ export type Either = Left | Right type DTO = Either<_E>, _A>> // FIXME: naming -// This runs on the host, what is passed into the worker at `pool.run` has to -// be serializable. export function fromWorkerPool(): F { - // I believe, by default, #workers = #cpu cores, which is probably what we want? + const l = os.cpus().length const pool = new Pool({ + // "Our testing has shown that a maxQueue size of approximately the square + // of the maximum number of threads is generally sufficient and performs + // well for many cases" + // + // via https://github.com/piscinajs/piscina#queue-size + maxQueue: l ** l, + maxThreads: l, // FIXME: get path dynamically filename: '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', @@ -26,15 +33,16 @@ export function fromWorkerPool(): F { return (payload) => pipe( - T.succeedWith(() => JSON.stringify(payload)), + // host -> worker + T.succeedWith(() => JSON.stringify(payload, null, 2)), T.chain((value) => T.promise(() => pool.run(value, { name: 'makeCacheItemFromFilePath' }))), + // worker -> host T.chain((value) => T.succeedWith(() => JSON.parse(value))), T.chain(({ _tag, value }) => T.if_( _tag === 'right', - // FIXME: effect () => T.succeedWith(() => value), - // FIXME: Signature claims it doesn't fail. + // Signature claims it doesn't fail, so we're just dying here. () => T.die(value), ), ), @@ -54,6 +62,6 @@ export function makeCacheItemFromFilePath(payload: string): Promise { provideConsole, provideDummyTracing, T.runPromise, - (p) => p.then((value) => JSON.stringify(value)), + (p) => p.then((value) => JSON.stringify(value, null, 2)), ) } From b4dcb709d732cea2697b13c36d30f78fc20bf47d Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 16:26:06 +0200 Subject: [PATCH 10/13] style: improve --- .../fetchData/makeCacheItemFromFilePath.worker.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index 86a9653e..c16f8781 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,7 +1,6 @@ import { provideDummyTracing } from '@contentlayer/utils' import { pipe, provideConsole, T } from '@contentlayer/utils/effect' import type { _A, _E } from '@effect-ts/core/Utils' -import fs from 'node:fs' import * as os from 'node:os' import Pool from 'piscina' @@ -9,23 +8,24 @@ import * as _ from './makeCacheItemFromFilePath.js' type F = typeof _.makeCacheItemFromFilePath -export type Left = { _tag: 'left'; value: E } -export type Right = { _tag: 'right'; value: A } +export type Left = { _tag: 'left'; value: E } +export type Right<_, A> = { _tag: 'right'; value: A } export type Either = Left | Right type DTO = Either<_E>, _A>> // FIXME: naming export function fromWorkerPool(): F { - const l = os.cpus().length + const cores = os.cpus().length const pool = new Pool({ // "Our testing has shown that a maxQueue size of approximately the square // of the maximum number of threads is generally sufficient and performs // well for many cases" // // via https://github.com/piscinajs/piscina#queue-size - maxQueue: l ** l, - maxThreads: l, + maxQueue: cores ** cores, + // by default, this is cores * 1.5 + maxThreads: cores, // FIXME: get path dynamically filename: '/home/ts/dev/code/contentlayer/packages/@contentlayer/source-files/dist/fetchData/makeCacheItemFromFilePath.worker.js', From fcb01033feada3f69cea1ff9ec7de4c223b3b204 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 20:57:44 +0200 Subject: [PATCH 11/13] feat: replace dummy with live tracing in worker --- .../src/fetchData/makeCacheItemFromFilePath.worker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index c16f8781..5ac2eb7a 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,4 +1,4 @@ -import { provideDummyTracing } from '@contentlayer/utils' +import { provideDummyTracing, provideJaegerTracing } from '@contentlayer/utils' import { pipe, provideConsole, T } from '@contentlayer/utils/effect' import type { _A, _E } from '@effect-ts/core/Utils' import * as os from 'node:os' @@ -60,7 +60,7 @@ export function makeCacheItemFromFilePath(payload: string): Promise { (value) => ({ _tag: 'right', value } as const), ), provideConsole, - provideDummyTracing, + provideJaegerTracing('worker'), T.runPromise, (p) => p.then((value) => JSON.stringify(value, null, 2)), ) From fbd13b3af471ee8f0f8fd142b88e08240fc94841 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Fri, 8 Jul 2022 21:40:38 +0200 Subject: [PATCH 12/13] build: unserialized plugin throwing --- examples/gatsbydocs/contentlayer.config.ts | 26 +++++++++---------- .../src/fetchData/fetchAllDocuments.ts | 5 ++-- .../fetchData/makeCacheItemFromFilePath.ts | 3 ++- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/gatsbydocs/contentlayer.config.ts b/examples/gatsbydocs/contentlayer.config.ts index 9d86bfeb..3bc2a4da 100644 --- a/examples/gatsbydocs/contentlayer.config.ts +++ b/examples/gatsbydocs/contentlayer.config.ts @@ -1,6 +1,6 @@ import rehypeShiki from '@stefanprobst/rehype-shiki' -import type { FieldDef} from 'contentlayer/source-files'; -import { defineDocumentType, defineNestedType,makeSource } from 'contentlayer/source-files' +import type { FieldDef } from 'contentlayer/source-files' +import { defineDocumentType, defineNestedType, makeSource } from 'contentlayer/source-files' import { createRequire } from 'module' import * as path from 'path' import * as shiki from 'shiki' @@ -20,7 +20,7 @@ const fields: Record = { }, jsdoc: { type: 'list', - of: {type: 'string'}, + of: { type: 'string' }, }, tableOfContentsDepth: { type: 'number' }, showTopLevelSignatures: { type: 'boolean' }, @@ -61,20 +61,20 @@ const Tutorial = defineDocumentType(() => ({ })) export default makeSource(async () => { - const require = createRequire(import.meta.url) - const shikiPkgPath = (dir: string) => path.join(require.resolve('shiki'), '..', '..', dir, path.sep) - const highlighter = await shiki.getHighlighter({ - paths: { languages: shikiPkgPath('languages'), themes: shikiPkgPath('themes') }, - theme: 'github-light', - }) + // const require = createRequire(import.meta.url) + // const shikiPkgPath = (dir: string) => path.join(require.resolve('shiki'), '..', '..', dir, path.sep) + // const highlighter = await shiki.getHighlighter({ + // paths: { languages: shikiPkgPath('languages'), themes: shikiPkgPath('themes') }, + // theme: 'github-light', + // }) return { contentDirPath: 'content', documentTypes: [Reference, HowTo, Conceptual, Tutorial], // onUnknownDocuments: 'skip-ignore', - markdown: { - // '@stefanprobst/rehype-shiki', {} - rehypePlugins: [[rehypeShiki, { highlighter }]], - }, + // markdown: { + // // '@stefanprobst/rehype-shiki', {} + // rehypePlugins: [[rehypeShiki, { highlighter }]], + // }, } }) diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index b50d6695..a888db6e 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -11,8 +11,8 @@ import { FetchDataError } from '../errors/index.js' import type { Flags } from '../index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import { DocumentTypeMap, provideDocumentTypeMapState } from './DocumentTypeMap.js' +import { makeCacheItemFromFilePath as withoutPool } from './makeCacheItemFromFilePath.js' import { fromWorkerPool } from './makeCacheItemFromFilePath.worker.js' -// import {createPool} from './makeCacheItemFromFilePath.worker.js' export const fetchAllDocuments = ({ coreSchemaDef, @@ -46,10 +46,10 @@ export const fetchAllDocuments = ({ const concurrencyLimit = os.cpus().length const makeCacheItemFromFilePath = fromWorkerPool() + // const makeCacheItemFromFilePath = withoutPool const { dataErrors, documents, documentTypeMap } = yield* $( pipe( allRelativeFilePaths, - // TODO: parallalize T.forEachParN(concurrencyLimit, (relativeFilePath) => makeCacheItemFromFilePath({ relativeFilePath, @@ -74,7 +74,6 @@ export const fetchAllDocuments = ({ } const res = These.result(a) if (E.isLeft(res)) { - // FIXME: type errors = Chunk.append_(errors, FetchDataError.fromSerialized(res.left as unknown as any)) } else { values = Chunk.append_(values, res.right.tuple[0]) diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts index 504c3c25..f84f30e9 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.ts @@ -10,7 +10,7 @@ import yaml from 'yaml' import { FetchDataError } from '../errors/index.js' import type { ContentTypeMap, FilePathPatternMap } from '../types.js' import { makeAndProvideDocumentContext } from './DocumentContext.js' -import type { DocumentTypeName, HasDocumentTypeMapState } from './DocumentTypeMap.js' +import type { DocumentTypeName } from './DocumentTypeMap.js' import { makeDocument } from './mapping.js' import type { RawContent, RawContentJSON, RawContentMarkdown, RawContentMDX, RawContentYAML } from './types.js' import { validateDocumentData } from './validateDocumentData.js' @@ -20,6 +20,7 @@ export const makeCacheItemFromFilePath = ({ filePathPatternMap, coreSchemaDef, contentDirPath, + // FIXME: This doesn't serialize properly. options, previousCache, contentTypeMap, From 5c827a33785f36f0817f2ee5c3143eee49bd0c43 Mon Sep 17 00:00:00 2001 From: Timm Stelzer Date: Sat, 9 Jul 2022 20:14:12 +0200 Subject: [PATCH 13/13] docs: update --- .../source-files/src/fetchData/fetchAllDocuments.ts | 2 +- .../src/fetchData/makeCacheItemFromFilePath.worker.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts index a888db6e..3a4e74a2 100644 --- a/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts +++ b/packages/@contentlayer/source-files/src/fetchData/fetchAllDocuments.ts @@ -2,7 +2,7 @@ import type * as core from '@contentlayer/core' import type { PosixFilePath } from '@contentlayer/utils' import { asMutableArray, posixFilePath } from '@contentlayer/utils' import type { HasConsole } from '@contentlayer/utils/effect' -import { Chunk, E, HashMap, O, Option, OT, pipe, T, These, Tp } from '@contentlayer/utils/effect' +import { Chunk, E, O, Option, OT, pipe, T, These, Tp } from '@contentlayer/utils/effect' import { fs } from '@contentlayer/utils/node' import glob from 'fast-glob' import * as os from 'node:os' diff --git a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts index 5ac2eb7a..1a10b45f 100644 --- a/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts +++ b/packages/@contentlayer/source-files/src/fetchData/makeCacheItemFromFilePath.worker.ts @@ -1,4 +1,4 @@ -import { provideDummyTracing, provideJaegerTracing } from '@contentlayer/utils' +import { provideJaegerTracing } from '@contentlayer/utils' import { pipe, provideConsole, T } from '@contentlayer/utils/effect' import type { _A, _E } from '@effect-ts/core/Utils' import * as os from 'node:os' @@ -14,6 +14,7 @@ export type Either = Left | Right type DTO = Either<_E>, _A>> +// TODO: generalize codec / serializing // FIXME: naming export function fromWorkerPool(): F { const cores = os.cpus().length @@ -24,7 +25,7 @@ export function fromWorkerPool(): F { // // via https://github.com/piscinajs/piscina#queue-size maxQueue: cores ** cores, - // by default, this is cores * 1.5 + // FIXME: on the default, (#cores * 1.5), this lead to memory issues that caused a thread panic maxThreads: cores, // FIXME: get path dynamically filename: @@ -34,7 +35,7 @@ export function fromWorkerPool(): F { return (payload) => pipe( // host -> worker - T.succeedWith(() => JSON.stringify(payload, null, 2)), + T.succeedWith(() => JSON.stringify(payload)), T.chain((value) => T.promise(() => pool.run(value, { name: 'makeCacheItemFromFilePath' }))), // worker -> host T.chain((value) => T.succeedWith(() => JSON.parse(value))), @@ -62,6 +63,6 @@ export function makeCacheItemFromFilePath(payload: string): Promise { provideConsole, provideJaegerTracing('worker'), T.runPromise, - (p) => p.then((value) => JSON.stringify(value, null, 2)), + (p) => p.then((value) => JSON.stringify(value)), ) }