Exclude anchor and query strung from URLs in sitemap connector (#86)

Author: ebrehault

package.json

@@ -1,6 +1,6 @@
 {
   "name": "nuclia-sync-agent-app",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "description": "This is a Nuclia Sync Agent App",
   "main": "build/index.js",
   "scripts": {

server/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 1.6.1 (2025-02-14)
+
+- Exclude anchor and query strung from URLs in sitemap connector
+
 # 1.6.0 (2025-01-28)
 
 - Support files download when using sitemap connector with local extraction

server/src/logic/connector/infrastructure/connectors/sitemap.connector.ts

@@ -26,7 +26,7 @@ export function parseSitemap(sitemapContent: string): Promise<SiteMapModel[]> {
     const $ = cheerio.load(sitemapContent, { xml: true });
 
     $('url').each((_, element) => {
-      const loc = $(element).find('loc').text();
+      const loc = $(element).find('loc').text().split('?')[0].split('#')[0];
       const lastmod = $(element).find('lastmod').text();
       urls.push({ loc, lastmod });
     });