Drop axios dependency (#3177)

* Minor adaptation of VerifyCidTransform implementation

* refactor: factorize content-encoding negotiation into new lib

* bsky: Use undici to stream blob

* fixup! bsky: Use undici to stream blob

* disable ssrf bsky protection in dev-env

* remove http requests to self to host "/img/"

* drop axios from tests

* fixes

* fix tests

* reviex changes

* properly handle HEAD requests

* handle client disconnection

* fix tests

* drop unrelated change

* tidy

* tidy

* tidy

* remove axios from dev-env

* remove axios from identity package

* use undici 6

* remove axios dependency from ozone

* tidy

* remove axios from PDS package

* avoid killing bsky-pds connections

* improve debugging data

* Better handle invalid CID

* tidy

* tidy

* refactor "allFulfilled" util in @atproto/common

* tidy

---------

Co-authored-by: devin ivy <[email protected]>

.changeset/cool-hairs-tap.md

@@ -0,0 +1,6 @@
+---
+"@atproto/bsky": patch
+---
+
+Improve performance when serving blobs
+

.changeset/eight-cobras-swim.md

@@ -0,0 +1,5 @@
+---
+"@atproto/common-web": patch
+---
+
+Add `createRetryable` utility function

.changeset/loud-ants-tap.md

@@ -0,0 +1,5 @@
+---
+"@atproto-labs/xrpc-utils": patch
+---
+
+New utility package to work with xrpc-server

.changeset/modern-bees-share.md

@@ -0,0 +1,5 @@
+---
+"@atproto-labs/did-resolver": patch
+---
+
+Ensure proper escaping when building PLC url

.changeset/neat-insects-judge.md

@@ -0,0 +1,9 @@
+---
+"@atproto/identity": patch
+"@atproto/dev-env": patch
+"@atproto/ozone": patch
+"@atproto/bsky": patch
+"@atproto/pds": patch
+---
+
+Remove dependency on Axios

.changeset/witty-owls-fail.md

@@ -0,0 +1,5 @@
+---
+"@atproto/common-web": patch
+---
+
+Add `allFulfilled` utility

package.json

@@ -36,7 +36,7 @@
     "@swc/core": "^1.3.42",
     "@swc/jest": "^0.2.24",
     "@types/jest": "^28.1.4",
-    "@types/node": "^18.19.56",
+    "@types/node": "^18.19.67",
     "@typescript-eslint/eslint-plugin": "^7.4.0",
     "@typescript-eslint/parser": "^7.4.0",
     "dotenv": "^16.0.3",

packages/aws/src/bunny.ts

@@ -1,4 +1,4 @@
-import { handleAllSettledErrors } from '@atproto/common'
+import { allFulfilled } from '@atproto/common'
 import { ImageInvalidator } from './types'
 
 export type BunnyConfig = {
@@ -11,15 +11,14 @@ const API_PURGE_URL = 'https://api.bunny.net/purge'
 export class BunnyInvalidator implements ImageInvalidator {
   constructor(public cfg: BunnyConfig) {}
   async invalidate(_subject: string, paths: string[]) {
-    const results = await Promise.allSettled(
+    await allFulfilled(
       paths.map(async (path) =>
         purgeUrl({
           url: this.cfg.urlPrefix + path,
           accessKey: this.cfg.accessKey,
         }),
       ),
     )
-    handleAllSettledErrors(results)
   }
 }
 

packages/aws/src/util.ts

@@ -1,14 +1,13 @@
-import { handleAllSettledErrors } from '@atproto/common'
+import { allFulfilled } from '@atproto/common'
 import { ImageInvalidator } from './types'
 
 export class MultiImageInvalidator implements ImageInvalidator {
   constructor(public invalidators: ImageInvalidator[]) {}
   async invalidate(subject: string, paths: string[]) {
-    const results = await Promise.allSettled(
+    await allFulfilled(
       this.invalidators.map((invalidator) =>
         invalidator.invalidate(subject, paths),
       ),
     )
-    handleAllSettledErrors(results)
   }
 }

packages/bsky/package.json

@@ -27,9 +27,12 @@
     "buf:gen": "buf generate ../bsync/proto && buf generate ./proto"
   },
   "dependencies": {
+    "@atproto-labs/fetch-node": "workspace:*",
+    "@atproto-labs/xrpc-utils": "workspace:*",
     "@atproto/api": "workspace:^",
     "@atproto/common": "workspace:^",
     "@atproto/crypto": "workspace:^",
+    "@atproto/did": "workspace:^",
     "@atproto/identity": "workspace:^",
     "@atproto/lexicon": "workspace:^",
     "@atproto/repo": "workspace:^",
@@ -41,6 +44,7 @@
     "@connectrpc/connect-express": "^1.1.4",
     "@connectrpc/connect-node": "^1.1.4",
     "@did-plc/lib": "^0.0.1",
+    "@types/http-errors": "^2.0.1",
     "compression": "^1.7.4",
     "cors": "^2.8.5",
     "express": "^4.17.2",
@@ -60,7 +64,8 @@
     "statsig-node": "^5.23.1",
     "structured-headers": "^1.0.1",
     "typed-emitter": "^2.1.0",
-    "uint8arrays": "3.0.0"
+    "uint8arrays": "3.0.0",
+    "undici": "^6.19.8"
   },
   "devDependencies": {
     "@atproto/api": "workspace:^",
@@ -76,7 +81,6 @@
     "@types/express-serve-static-core": "^4.17.36",
     "@types/pg": "^8.6.6",
     "@types/qs": "^6.9.7",
-    "axios": "^0.27.2",
     "jest": "^28.1.2",
     "ts-node": "^10.8.2",
     "typescript": "^5.6.3"

packages/bsky/src/api/blob-dispatcher.ts

@@ -0,0 +1,38 @@
+import { isUnicastIp, unicastLookup } from '@atproto-labs/fetch-node'
+import { Agent, Dispatcher, Pool, RetryAgent } from 'undici'
+
+import { ServerConfig } from '../config'
+import { RETRYABLE_HTTP_STATUS_CODES } from '../util/retry'
+
+export function createBlobDispatcher(cfg: ServerConfig): Dispatcher {
+  const baseDispatcher = new Agent({
+    allowH2: cfg.proxyAllowHTTP2, // This is experimental
+    headersTimeout: cfg.proxyHeadersTimeout,
+    maxResponseSize: cfg.proxyMaxResponseSize,
+    bodyTimeout: cfg.proxyBodyTimeout,
+    factory: cfg.disableSsrfProtection
+      ? undefined
+      : (origin, opts) => {
+          const { protocol, hostname } =
+            origin instanceof URL ? origin : new URL(origin)
+          if (protocol !== 'https:') {
+            throw new Error(`Forbidden protocol "${protocol}"`)
+          }
+          if (isUnicastIp(hostname) === false) {
+            throw new Error('Hostname resolved to non-unicast address')
+          }
+          return new Pool(origin, opts)
+        },
+    connect: {
+      lookup: cfg.disableSsrfProtection ? undefined : unicastLookup,
+    },
+  })
+
+  return cfg.proxyMaxRetries > 0
+    ? new RetryAgent(baseDispatcher, {
+        statusCodes: [...RETRYABLE_HTTP_STATUS_CODES],
+        methods: ['GET', 'HEAD'],
+        maxRetries: cfg.proxyMaxRetries,
+      })
+    : baseDispatcher
+}