diff --git a/core/audits/seo/robots-txt.js b/core/audits/seo/robots-txt.js index 7176c473d643..19450c88b64e 100644 --- a/core/audits/seo/robots-txt.js +++ b/core/audits/seo/robots-txt.js @@ -27,7 +27,7 @@ const DIRECTIVE_SAFELIST = new Set([ DIRECTIVE_ALLOW, DIRECTIVE_SITEMAP, // universally supported 'crawl-delay', // yahoo, bing, yandex 'clean-param', 'host', // yandex - 'request-rate', 'visit-time', 'noindex', // not officially supported, but used in the wild + 'request-rate', 'visit-time', 'noindex', 'content-signal', // not officially supported, but used in the wild ]); const SITEMAP_VALID_PROTOCOLS = new Set(['https:', 'http:', 'ftp:']); diff --git a/core/test/audits/seo/robots-txt-test.js b/core/test/audits/seo/robots-txt-test.js index 7de1fa59816c..73a3676bd891 100644 --- a/core/test/audits/seo/robots-txt-test.js +++ b/core/test/audits/seo/robots-txt-test.js @@ -224,6 +224,9 @@ User-agent: NotOfficial noindex: /bla Visit-time: 0600-0845 Request-rate: 1/30m + +User-agent: RespectfulAI +Content-Signal: ai-train=yes, search=yes, ai-input=yes `, }, ];