Custom User-Agent headers

#162
dahlia · Nov 4, 2024 · cfe509f · cfe509f
1 parent e6a416f
commit cfe509f
Show file tree

Hide file tree

Showing 20 changed files with 863 additions and 537 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -8,9 +8,29 @@ Version 1.3.0
 
 To be released.
 
- -  Fedify now makes HTTP requests with the proper `User-Agent` header.
+ -  Fedify now makes HTTP requests with the proper `User-Agent` header. [[#162]]
 
      -  Added `getUserAgent()` function.
+     -  Added `GetUserAgentOptions` interface.
+     -  Added `getDocumentLoader()` function.
+     -  Added `GetDocumentLoaderOptions` interface.
+     -  The type of `getAuthenticatedDocumentLoader()` function's second
+        parameter became `GetAuthenticatedDocumentLoaderOptions | undefined`
+        (was `boolean | undefined`).
+     -  Added `GetAuthenticatedDocumentLoaderOptions` interface.
+     -  Deprecated `fetchDocumentLoader()` function.
+     -  Added `LookupObjectOptions.userAgent` option.
+     -  Added the type of `getActorHandle()` function's second parameter became
+        `GetActorHandleOptions | undefined` (was `NormalizeActorHandleOptions |
+        undefined`).
+     -  Added `GetActorHandleOptions` interface.
+     -  Added the optional second parameter to `lookupWebFinger()` function.
+     -  Added `LookupWebFingerOptions` interface.
+     -  Added `GetNodeInfoOptions.userAgent` option.
+     -  Added `-u`/--user-agent` option to `fedify lookup` subcommand.
+     -  Added `-u`/--user-agent` option to `fedify node` subcommand.
+
+[#162]: https://github.com/dahlia/fedify/issues/162
 
 
 Version 1.2.2

diff --git a/cli/docloader.ts b/cli/docloader.ts
@@ -1,19 +1,25 @@
 import {
   type DocumentLoader,
-  fetchDocumentLoader,
+  getDocumentLoader as getDefaultDocumentLoader,
   kvCache,
 } from "@fedify/fedify";
 import { DenoKvStore } from "@fedify/fedify/x/denokv";
 import { join } from "@std/path";
 import { getCacheDir } from "./cache.ts";
 
-let documentLoader: DocumentLoader | undefined = undefined;
+const documentLoaders: Record<string, DocumentLoader> = {};
 
-export async function getDocumentLoader(): Promise<DocumentLoader> {
-  if (documentLoader) return documentLoader;
+export interface DocumentLoaderOptions {
+  userAgent?: string;
+}
+
+export async function getDocumentLoader(
+  { userAgent }: DocumentLoaderOptions = {},
+): Promise<DocumentLoader> {
+  if (documentLoaders[userAgent ?? ""]) return documentLoaders[userAgent ?? ""];
   const path = join(await getCacheDir(), "kv");
   const kv = new DenoKvStore(await Deno.openKv(path));
-  return documentLoader = kvCache({
+  return documentLoaders[userAgent ?? ""] = kvCache({
     kv,
     rules: [
       [
@@ -50,12 +56,15 @@ export async function getDocumentLoader(): Promise<DocumentLoader> {
         Temporal.Duration.from({ seconds: 0 }),
       ],
     ],
-    loader(url) {
-      return fetchDocumentLoader(url, true);
-    },
+    loader: getDefaultDocumentLoader({
+      allowPrivateAddress: true,
+      userAgent,
+    }),
   });
 }
 
-export function getContextLoader(): Promise<DocumentLoader> {
-  return getDocumentLoader();
+export function getContextLoader(
+  options: DocumentLoaderOptions = {},
+): Promise<DocumentLoader> {
+  return getDocumentLoader(options);
 }
diff --git a/cli/lookup.ts b/cli/lookup.ts
@@ -31,14 +31,19 @@ export const command = new Command()
   .option("-e, --expand", "Expand the fetched JSON-LD document.", {
     conflicts: ["raw", "compact"],
   })
+  .option("-u, --user-agent <string>", "The custom User-Agent header value.")
   .action(async (options, url: string) => {
     const spinner = ora({
       text: "Looking up the object...",
       discardStdin: false,
     }).start();
     let server: TemporaryServer | undefined = undefined;
-    const documentLoader = await getDocumentLoader();
-    const contextLoader = await getContextLoader();
+    const documentLoader = await getDocumentLoader({
+      userAgent: options.userAgent,
+    });
+    const contextLoader = await getContextLoader({
+      userAgent: options.userAgent,
+    });
     let authLoader: DocumentLoader | undefined = undefined;
     if (options.authorizedFetch) {
       spinner.text = "Generating a one-time key pair...";
@@ -87,7 +92,11 @@ export const command = new Command()
       spinner.text = "Looking up the object...";
       const object = await lookupObject(
         url,
-        { documentLoader: authLoader ?? documentLoader, contextLoader },
+        {
+          documentLoader: authLoader ?? documentLoader,
+          contextLoader,
+          userAgent: options.userAgent,
+        },
       );
       spinner.succeed();
       if (object == null) {

diff --git a/cli/node.ts b/cli/node.ts
@@ -1,6 +1,6 @@
 import { colors } from "@cliffy/ansi";
 import { Command } from "@cliffy/command";
-import { formatSemVer, getNodeInfo } from "@fedify/fedify";
+import { formatSemVer, getNodeInfo, getUserAgent } from "@fedify/fedify";
 import { createJimp } from "@jimp/core";
 import webp from "@jimp/wasm-webp";
 import { getLogger } from "@logtape/logtape";
@@ -34,14 +34,18 @@ export const command = new Command()
     "Print metadata fields of the NodeInfo document.",
     { conflicts: ["raw"] },
   )
+  .option("-u, --user-agent <string>", "The custom User-Agent header value.")
   .action(async (options, host: string) => {
     const spinner = ora({
       text: "Fetching a NodeInfo document...",
       discardStdin: false,
     }).start();
     const url = new URL(URL.canParse(host) ? host : `https://${host}/`);
     if (options.raw) {
-      const nodeInfo = await getNodeInfo(url, { parse: "none" });
+      const nodeInfo = await getNodeInfo(url, {
+        parse: "none",
+        userAgent: options.userAgent,
+      });
       if (nodeInfo === undefined) {
         spinner.fail("No NodeInfo document found.");
         console.error("No NodeInfo document found.");
@@ -53,6 +57,7 @@ export const command = new Command()
     }
     const nodeInfo = await getNodeInfo(url, {
       parse: options.bestEffort ? "best-effort" : "strict",
+      userAgent: options.userAgent,
     });
     logger.debug("NodeInfo document: {nodeInfo}", { nodeInfo });
     if (nodeInfo == undefined) {
@@ -70,8 +75,14 @@ export const command = new Command()
     if (options.favicon) {
       spinner.text = "Fetching the favicon...";
       try {
-        const faviconUrl = await getFaviconUrl(url);
-        const response = await fetch(faviconUrl);
+        const faviconUrl = await getFaviconUrl(url, options.userAgent);
+        const response = await fetch(faviconUrl, {
+          headers: {
+            "User-Agent": options.userAgent == null
+              ? getUserAgent()
+              : options.userAgent,
+          },
+        });
         if (response.ok) {
           const contentType = response.headers.get("Content-Type");
           let buffer: ArrayBuffer = await response.arrayBuffer();
@@ -208,8 +219,15 @@ const LINK_REGEXP =
   /<link((?:\s+(?:[-a-z]+)=(?:"[^"]*"|'[^']*'|[^\s]+))*)\s*\/?>/ig;
 const LINK_ATTRS_REGEXP = /(?:\s+([-a-z]+)=("[^"]*"|'[^']*'|[^\s]+))/ig;
 
-async function getFaviconUrl(url: string | URL): Promise<URL> {
-  const response = await fetch(url);
+async function getFaviconUrl(
+  url: string | URL,
+  userAgent?: string,
+): Promise<URL> {
+  const response = await fetch(url, {
+    headers: {
+      "User-Agent": userAgent == null ? getUserAgent() : userAgent,
+    },
+  });
   const text = await response.text();
   for (const match of text.matchAll(LINK_REGEXP)) {
     const attrs: Record<string, string> = {};

diff --git a/docs/cli.md b/docs/cli.md
@@ -639,6 +639,20 @@ Person {
 }
 ~~~~
 
+### `-u`/`--user-agent`: Custom `User-Agent` header
+
+*This option is available since Fedify 1.3.0.*
+
+By default, the `fedify lookup` command sends the `User-Agent` header with the
+value `Fedify/1.3.0 (Deno/2.0.4)` (version numbers may vary).  You can specify
+a custom `User-Agent` header by using the `-u`/`--user-agent` option.  For
+example, to send the `User-Agent` header with the value `MyApp/1.0`, run the
+below command:
+
+~~~~ sh
+fedify lookup --user-agent MyApp/1.0 @[email protected]
+~~~~
+
 
 `fedify inbox`: Ephemeral inbox server
 --------------------------------------
@@ -797,6 +811,20 @@ instance.
 The `-m`/`--metadata` option is used to show the extra metadata of the NodeInfo,
 i.e., the `metadata` field of the document.
 
+### `-u`/`--user-agent`: Custom `User-Agent` header
+
+*This option is available since Fedify 1.3.0.*
+
+By default, the `fedify node` command sends the `User-Agent` header with the
+value `Fedify/1.3.0 (Deno/2.0.4)` (version numbers may vary).  You can specify
+a custom `User-Agent` header by using the `-u`/`--user-agent` option.  For
+example, to send the `User-Agent` header with the value `MyApp/1.0`, run the
+below command:
+
+~~~~ sh
+fedify node --user-agent MyApp/1.0 mastodon.social
+~~~~
+
 
 `fedify tunnel`: Exposing a local HTTP server to the public internet
 --------------------------------------------------------------------

diff --git a/docs/manual/federation.md b/docs/manual/federation.md
@@ -205,6 +205,47 @@ Turned off by default.
 
 [SSRF]: https://owasp.org/www-community/attacks/Server_Side_Request_Forgery
 
+### `userAgent`
+
+*This API is available since Fedify 1.3.0.*
+
+The options for making `User-Agent` header in the HTTP requests that Fedify
+makes.  By default, it contains the name and version of the Fedify library,
+and the name and version of the JavaScript runtime, e.g.:
+
+~~~~
+Fedify/1.3.0 (Deno/2.0.4)
+Fedify/1.3.0 (Node.js/v22.10.0)
+Fedify/1.3.0 (Bun/1.1.33)
+~~~~
+
+You can customize the `User-Agent` header by providing options like `software`
+and `url`.  For example, if you provide the following options:
+
+~~~~ ts
+{
+  software: "MyApp/1.0.0",
+  url: "https://myinstance.com/"
+}
+~~~~
+
+The `User-Agent` header will be like:
+
+~~~~
+MyApp/1.0.0 (Fedify/1.3.0; Deno/2.0.4; +https://myinstance.com/)
+~~~~
+
+Or, you can rather provide a custom `User-Agent` string directly instead of
+an object for options.
+
+> [!CAUTION]
+>
+> This settings do not affect the `User-Agent` header of the HTTP requests
+> that `lookupWebFinger()`, `lookupObject()`, and `getNodeInfo()` functions make,
+> since they do not depend on the `Federation` object.
+>
+> However, `Context.lookupObject()` method is affected by this settings.
+
 ### `outboxRetryPolicy`
 
 *This API is available since Fedify 0.12.0.*