Feature(sitemap): named files chunking strategy

.changeset/floppy-times-grab.md

@@ -0,0 +1,36 @@
+---
+'@astrojs/sitemap': minor
+---
+
+Adds the ability to split sitemap generation into chunks based on customizable logic. This allows for better management of large sitemaps and improved performance. The new `chunks` option in the sitemap configuration allows users to define functions that categorize sitemap items into different chunks. Each chunk is then written to a separate sitemap file. 
+
+```
+integrations: [
+  sitemap({
+    serialize(item) { th
+      return item
+    },
+    chunks: { // this property will be treated last on the configuration
+      'blog': (item) => {  // will produce a sitemap file with `blog` name (sitemap-blog-0.xml)
+        if (/blog/.test(item.url)) { // filter path that will be included in this specific sitemap file 
+          item.changefreq = 'weekly';
+          item.lastmod = new Date();
+          item.priority = 0.9; // define specific properties for this filtered path
+          return item;
+        }
+      },
+      'glossary': (item) => {
+        if (/glossary/.test(item.url)) {
+          item.changefreq = 'weekly';
+          item.lastmod = new Date();
+          item.priority = 0.7;
+          return item;
+        }
+      }
+
+      // the rest of the path will be stored in `sitemap-pages.0.xml`
+    },
+  }),
+],
+
+  ```

package.json

@@ -83,10 +83,10 @@
       ]
     },
     "onlyBuiltDependencies": [
-      "esbuild",
-      "workerd",
       "@biomejs/biome",
-      "sharp"
+      "esbuild",
+      "sharp",
+      "workerd"
     ]
   }
 }

packages/integrations/sitemap/src/index.ts

@@ -7,8 +7,10 @@ import { ZodError } from 'zod';
 import { generateSitemap } from './generate-sitemap.js';
 import { validateOptions } from './validate-options.js';
 import { writeSitemap } from './write-sitemap.js';
+import { writeSitemapChunk } from './write-sitemap-chunk.js';
 
 export { EnumChangefreq as ChangeFreqEnum } from 'sitemap';
+
 export type ChangeFreq = `${EnumChangefreq}`;
 export type SitemapItem = Pick<
 	SitemapItemLoose,
@@ -18,36 +20,35 @@ export type LinkItem = LinkItemBase;
 
 export type SitemapOptions =
 	| {
-			filenameBase?: string;
-			filter?(page: string): boolean;
-			customSitemaps?: string[];
-			customPages?: string[];
-
-			i18n?: {
-				defaultLocale: string;
-				locales: Record<string, string>;
-			};
-			// number of entries per sitemap file
-			entryLimit?: number;
-
-			// sitemap specific
-			changefreq?: ChangeFreq;
-			lastmod?: Date;
-			priority?: number;
-
-			// called for each sitemap item just before to save them on disk, sync or async
-			serialize?(item: SitemapItem): SitemapItem | Promise<SitemapItem | undefined> | undefined;
-
-			xslURL?: string;
-
-			// namespace configuration
-			namespaces?: {
-				news?: boolean;
-				xhtml?: boolean;
-				image?: boolean;
-				video?: boolean;
-			};
-	  }
+		filenameBase?: string;
+		filter?(page: string): boolean;
+		customSitemaps?: string[];
+		customPages?: string[];
+
+		i18n?: {
+			defaultLocale: string;
+			locales: Record<string, string>;
+		};
+		// number of entries per sitemap file
+		entryLimit?: number;
+		// sitemap specific
+		changefreq?: ChangeFreq;
+		lastmod?: Date;
+		priority?: number;
+
+		// called for each sitemap item just before to save them on disk, sync or async
+		serialize?(item: SitemapItem): SitemapItem | Promise<SitemapItem | undefined> | undefined;
+
+		xslURL?: string;
+		chunks?: Record<string, (item: SitemapItem) => SitemapItem | Promise<SitemapItem | undefined> | undefined>
+		// namespace configuration
+		namespaces?: {
+			news?: boolean;
+			xhtml?: boolean;
+			image?: boolean;
+			video?: boolean;
+		};
+	}
 	| undefined;
 
 function formatConfigErrorMessage(err: ZodError) {
@@ -97,8 +98,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => {
 
 					const opts = validateOptions(config.site, options);
 
-					const { filenameBase, filter, customPages, customSitemaps, serialize, entryLimit } = opts;
-
+					const { filenameBase, filter, customPages, customSitemaps, serialize, entryLimit, chunks } = opts;
 					const outFile = `${filenameBase}-index.xml`;
 					const finalSiteUrl = new URL(config.base, config.site);
 					const shouldIgnoreStatus = isStatusCodePage(Object.keys(opts.i18n?.locales ?? {}));
@@ -174,9 +174,53 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => {
 							return;
 						}
 					}
+
 					const destDir = fileURLToPath(dir);
 					const lastmod = opts.lastmod?.toISOString();
 					const xslURL = opts.xslURL ? new URL(opts.xslURL, finalSiteUrl).href : undefined;
+
+					if (chunks) {
+						try {
+							let groupedUrlCollection: SitemapItem['url'][] = []
+							const chunksItem: Record<string, SitemapItem[]> = {};
+							for (const [key, cb] of Object.entries(chunks)) {
+								// Create a new, separate collection for each key
+								const collection: SitemapItem[] = [];
+
+								for (const item of urlData) {
+									// Await the asynchronous operation
+									const collect = await Promise.resolve(cb(item));
+									if (collect) {
+										collection.push(collect);
+									}
+								}
+
+								// Assign the specific collection to its key
+								chunksItem[key] = collection;
+								groupedUrlCollection = [...groupedUrlCollection, ...collection.map((coll) => coll.url)]
+							}
+							chunksItem['pages'] = urlData.filter((urlDataItem) => !(groupedUrlCollection.includes(urlDataItem.url)))
+								// Process each chunk here
+								await writeSitemapChunk({
+									filenameBase,
+									hostname: finalSiteUrl.href,
+									sitemapHostname: finalSiteUrl.href,
+									sourceData: chunksItem,
+									destinationDir: destDir,
+									publicBasePath: config.base,
+									customSitemaps,
+									limit: entryLimit,
+									xslURL,
+									lastmod,
+									namespaces: opts.namespaces,
+								}, config);
+								logger.info(`\`${outFile}\` created at \`${path.relative(process.cwd(), destDir)}\``);
+								return
+						} catch (err) {
+							logger.error(`Error chunking sitemaps\n${(err as any).toString()}`);
+							return;
+						}
+					}
 					await writeSitemap(
 						{
 							filenameBase: filenameBase,

packages/integrations/sitemap/src/schema.ts

@@ -47,6 +47,7 @@ export const SitemapOptionsSchema = z
 			})
 			.optional()
 			.default(SITEMAP_CONFIG_DEFAULTS.namespaces),
-	})
+		chunks: z.record(z.function().args(z.any()).returns(z.any())).optional(),
+		})
 	.strict()
 	.default(SITEMAP_CONFIG_DEFAULTS);

packages/integrations/sitemap/src/write-sitemap-chunk.ts

@@ -0,0 +1,128 @@
+import { createWriteStream, type WriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { normalize, resolve } from 'node:path';
+import { pipeline, Readable } from 'node:stream';
+import { promisify } from 'node:util';
+import type { AstroConfig } from 'astro';
+import { SitemapAndIndexStream, SitemapIndexStream, SitemapStream } from 'sitemap';
+import replace from 'stream-replace-string';
+import type { SitemapItem } from './index.js';
+
+
+type WriteSitemapChunkConfig = {
+	filenameBase: string;
+	hostname: string;
+	sitemapHostname?: string;
+	sourceData: Record<string, SitemapItem[]>;
+	destinationDir: string;
+	customSitemaps?: string[];
+	publicBasePath?: string;
+	limit?: number;
+	xslURL?: string;
+	lastmod?: string;
+	namespaces?: {
+		news?: boolean;
+		xhtml?: boolean;
+		image?: boolean;
+		video?: boolean;
+	};
+};
+
+// adapted from sitemap.js/sitemap-simple
+export async function writeSitemapChunk(
+	{
+		filenameBase,
+		hostname,
+		sitemapHostname = hostname,
+		sourceData,
+		destinationDir,
+		limit = 50000,
+		customSitemaps = [],
+		publicBasePath = './',
+		xslURL: xslUrl,
+		lastmod,
+		namespaces = { news: true, xhtml: true, image: true, video: true },
+	}: WriteSitemapChunkConfig,
+	astroConfig: AstroConfig,
+) {
+	await mkdir(destinationDir, { recursive: true });
+
+	// Normalize publicBasePath
+	let normalizedPublicBasePath = publicBasePath;
+	if (!normalizedPublicBasePath.endsWith('/')) {
+		normalizedPublicBasePath += '/';
+	}
+
+	// Array to collect all sitemap URLs for the index
+	const sitemapUrls: Array<{ url: string; lastmod?: string }> = [];
+
+	// Process each chunk separately
+	for (const [chunkName, items] of Object.entries(sourceData)) {
+		const sitemapAndIndexStream = new SitemapAndIndexStream({
+			limit,
+			xslUrl,
+			getSitemapStream: (i) => {
+				const sitemapStream = new SitemapStream({
+					hostname,
+					xslUrl,
+					// Custom namespace handling
+					xmlns: {
+						news: namespaces?.news !== false,
+						xhtml: namespaces?.xhtml !== false,
+						image: namespaces?.image !== false,
+						video: namespaces?.video !== false,
+					},
+				});
+
+				const path = `./${filenameBase}-${chunkName}-${i}.xml`;
+				const writePath = resolve(destinationDir, path);
+				const publicPath = normalize(normalizedPublicBasePath + path);
+
+				let stream: WriteStream;
+				if (astroConfig.trailingSlash === 'never' || astroConfig.build.format === 'file') {
+					// workaround for trailing slash issue in sitemap.js
+					const host = hostname.endsWith('/') ? hostname.slice(0, -1) : hostname;
+					const searchStr = `<loc>${host}/</loc>`;
+					const replaceStr = `<loc>${host}</loc>`;
+					stream = sitemapStream
+						.pipe(replace(searchStr, replaceStr))
+						.pipe(createWriteStream(writePath));
+				} else {
+					stream = sitemapStream.pipe(createWriteStream(writePath));
+				}
+
+				const url = new URL(publicPath, sitemapHostname).toString();
+
+				// Collect this sitemap URL for the index
+				sitemapUrls.push({ url, lastmod });
+
+				return [{ url, lastmod }, sitemapStream, stream];
+			},
+		});
+
+		// Create a readable stream from this chunk's items
+		const dataStream = Readable.from(items);
+
+		// Write this chunk's sitemap(s)
+		await promisify(pipeline)(dataStream, sitemapAndIndexStream);
+	}
+
+	// Now create the sitemap index with all the generated sitemaps
+	const indexStream = new SitemapIndexStream({ xslUrl });
+	const indexPath = resolve(destinationDir, `./${filenameBase}-index.xml`);
+	const indexWriteStream = createWriteStream(indexPath);
+
+	// Add custom sitemaps to the index
+	for (const url of customSitemaps) {
+		indexStream.write({ url, lastmod });
+	}
+
+	// Add all generated sitemaps to the index
+	for (const sitemapUrl of sitemapUrls) {
+		indexStream.write(sitemapUrl);
+	}
+
+	indexStream.end();
+
+	return await promisify(pipeline)(indexStream, indexWriteStream);
+}

packages/integrations/sitemap/src/write-sitemap.ts

@@ -93,7 +93,7 @@ export async function writeSitemap(
 			sitemapAndIndexStream,
 			{ url, lastmod },
 			'utf8',
-			() => {},
+			() => { },
 		);
 	}
 	return promisify(pipeline)(src, sitemapAndIndexStream, createWriteStream(indexPath));