feat: redis gateway

Author: didinele

packages/redis-gateway/README.md

@@ -22,18 +22,52 @@
 
 ## Usage
 
-Quickly spin up an instance:
+Set up an `.env` file:
+
+```
+REDIS_URL=redis://localhost:6379
+DISCORD_TOKEN=your-token-here
+DISCORD_PROXY_URL=htt://localhost:8080 # if you want to use an HTTP proxy for DAPI calls (optional)
+INTENTS=0 # intents to use (optional, defaults to none)
+SHARD_COUNT=1 # number of total shards your bot should be running (optional, defaults to Discord recommended count)
+SHARD_IDS=0 # comma-separated list of shard IDs to run (optional, defaults to all shards)
+SHARDS_PER_WORKER=2 # number of shards per worker_thread or "all" (optional, if not specified, all shards will be run in the main thread)
+```
 
-<!-- TODO: args -->
+Quickly spin up an instance:
 
-`docker run -d --restart unless-stopped --name gateway discordjs/redis-gateway`
+`docker run -d --restart unless-stopped --env-file .env --name gateway discordjs/redis-gateway`
 
 Use it:
 
-```ts
-// TODO
+```js
+import Redis from 'ioredis';
+import { PubSubRedisBroker } from '@discordjs/brokers';
+import { GatewayDispatchEvents } from 'discord-api-types/v10';
+
+const redis = new Redis();
+const broker = new PubSubRedisBroker({ redisClient: redis, encode, decode });
+
+broker.on(GatewayDispatchEvents.InteractionCreate, async ({ data: interaction, ack }) => {
+	if (interaction.type !== InteractionType.ApplicationCommand) {
+		return;
+	}
+
+	if (interaction.data.name === 'ping') {
+		// reply with pong using your favorite Discord API library
+	}
+
+	await ack();
+});
 ```
 
+For TypeScript usage, you can pass in a gereric type to the `PubSubRedisBroker` to map out all the events,
+refer to [this container's implementation](https://github.com/discordjs/discord.js/tree/main/packages/redis-gateway/src/index.ts#L15) for reference.
+
+Also note that [core](https://github.com/discordjs/discord.js/tree/main/packages/core) supports an
+abstract `gateway` property that can be easily implemented, making this pretty comfortable to
+use in conjunction. Refer to the [Gateway documentation](https://discord.js.org/docs/packages/core/main/Gateway:Interface)
+
 ## Links
 
 - [Website][website] ([source][website-source])

packages/redis-gateway/package.json

@@ -44,7 +44,13 @@
 	},
 	"homepage": "https://discord.js.org",
 	"dependencies": {
-		"tslib": "^2.5.0"
+		"@discordjs/brokers": "workspace:^",
+		"@discordjs/rest": "workspace:^",
+		"@discordjs/ws": "workspace:^",
+		"discord-api-types": "^0.37.41",
+		"ioredis": "^5.3.2",
+		"tslib": "^2.5.0",
+		"undici": "^5.22.0"
 	},
 	"devDependencies": {
 		"@types/node": "16.18.25",

packages/redis-gateway/src/discordEvents.ts

@@ -0,0 +1,15 @@
+import type { GatewayDispatchEvents, GatewayDispatchPayload, GatewaySendPayload } from 'discord-api-types/v10';
+
+// need this to be its own type for some reason, the compiler doesn't behave the same way if we in-line it
+type _DiscordEvents = {
+	[K in GatewayDispatchEvents]: GatewayDispatchPayload & {
+		t: K;
+	};
+};
+
+export type DiscordEvents = {
+	// @ts-expect-error - unclear why this ignore is needed, might be because dapi-types is missing some events from the union again
+	[K in keyof _DiscordEvents]: _DiscordEvents[K]['d'];
+} & {
+	gateway_send: GatewaySendPayload;
+};

packages/redis-gateway/src/env.ts

@@ -0,0 +1,35 @@
+import process from 'node:process';
+import type { GatewayIntentBits } from 'discord-api-types/v10';
+
+export class Env {
+	public readonly redisUrl: string = process.env.REDIS_URL!;
+
+	public readonly discordToken: string = process.env.DISCORD_TOKEN!;
+
+	public readonly discordProxyURL: string | null = process.env.DISCORD_PROXY_URL ?? null;
+
+	public readonly intents: GatewayIntentBits | 0 = Number(process.env.INTENTS ?? 0);
+
+	public readonly shardCount: number | null = process.env.SHARD_COUNT ? Number(process.env.SHARD_COUNT) : null;
+
+	public readonly shardIds: number[] | null = process.env.SHARD_IDS
+		? process.env.SHARD_IDS.split(',').map(Number)
+		: null;
+
+	public readonly shardsPerWorker: number | 'all' | null =
+		process.env.SHARDS_PER_WORKER === 'all'
+			? 'all'
+			: process.env.SHARDS_PER_WORKER
+			? Number(process.env.SHARDS_PER_WORKER)
+			: null;
+
+	private readonly REQUIRED_ENV_VARS = ['REDIS_URL', 'DISCORD_TOKEN'] as const;
+
+	public constructor() {
+		for (const key of this.REQUIRED_ENV_VARS) {
+			if (!(key in process.env)) {
+				throw new Error(`Missing required environment variable: ${key}`);
+			}
+		}
+	}
+}

packages/redis-gateway/src/index.ts

@@ -1 +1,62 @@
-console.log('Hello, from @discordjs/redis-gateway');
+import { randomBytes } from 'node:crypto';
+import { PubSubRedisBroker } from '@discordjs/brokers';
+import type { RESTOptions } from '@discordjs/rest';
+import { REST } from '@discordjs/rest';
+import type { OptionalWebSocketManagerOptions, RequiredWebSocketManagerOptions } from '@discordjs/ws';
+import { WorkerShardingStrategy, CompressionMethod, WebSocketManager, WebSocketShardEvents } from '@discordjs/ws';
+import Redis from 'ioredis';
+import { ProxyAgent } from 'undici';
+import type { DiscordEvents } from './discordEvents.js';
+import { Env } from './env.js';
+
+const env = new Env();
+
+const redisClient = new Redis(env.redisUrl);
+const broker = new PubSubRedisBroker<DiscordEvents>({
+	redisClient,
+});
+
+const restOptions: Partial<RESTOptions> = {};
+if (env.discordProxyURL) {
+	restOptions.api = `${env.discordProxyURL}/api`;
+}
+
+const rest = new REST(restOptions).setToken(env.discordToken);
+if (env.discordProxyURL) {
+	rest.setAgent(new ProxyAgent(env.discordProxyURL));
+}
+
+const gatewayOptions: Partial<OptionalWebSocketManagerOptions> & RequiredWebSocketManagerOptions = {
+	token: env.discordToken,
+	rest,
+	intents: env.intents,
+	compression: CompressionMethod.ZlibStream,
+	shardCount: env.shardCount,
+	shardIds: env.shardIds,
+};
+if (env.shardsPerWorker) {
+	gatewayOptions.buildStrategy = (manager) =>
+		new WorkerShardingStrategy(manager, { shardsPerWorker: env.shardsPerWorker! });
+}
+
+const gateway = new WebSocketManager(gatewayOptions);
+
+gateway
+	.on(WebSocketShardEvents.Debug, ({ message, shardId }) => console.log(`[WS Shard ${shardId}] [DEBUG]`, message))
+	.on(WebSocketShardEvents.Hello, ({ shardId }) => console.log(`[WS Shard ${shardId}] [HELLO]`))
+	.on(WebSocketShardEvents.Ready, ({ shardId }) => console.log(`[WS Shard ${shardId}] [READY]`))
+	.on(WebSocketShardEvents.Resumed, ({ shardId }) => console.log(`[WS Shard ${shardId}] [RESUMED]`))
+	.on(WebSocketShardEvents.Dispatch, ({ data }) => void broker.publish(data.t, data.d));
+
+broker.on('gateway_send', async ({ data, ack }) => {
+	for (const shardId of await gateway.getShardIds()) {
+		await gateway.send(shardId, data);
+	}
+
+	await ack();
+});
+
+// we use a random group name because we don't want work-balancing,
+// we need this to be fanned out so all shards get the payload
+await broker.subscribe(randomBytes(16).toString('hex'), ['gateway_send']);
+await gateway.connect();

yarn.lock

@@ -2019,7 +2019,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@discordjs/brokers@workspace:packages/brokers":
+"@discordjs/brokers@workspace:^, @discordjs/brokers@workspace:packages/brokers":
   version: 0.0.0-use.local
   resolution: "@discordjs/brokers@workspace:packages/brokers"
   dependencies:
@@ -2317,16 +2317,22 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@discordjs/redis-gateway@workspace:packages/redis-gateway"
   dependencies:
+    "@discordjs/brokers": "workspace:^"
+    "@discordjs/rest": "workspace:^"
+    "@discordjs/ws": "workspace:^"
     "@types/node": 16.18.25
     cross-env: ^7.0.3
+    discord-api-types: ^0.37.41
     eslint: ^8.39.0
     eslint-config-neon: ^0.1.46
     eslint-formatter-pretty: ^5.0.0
+    ioredis: ^5.3.2
     prettier: ^2.8.8
     tslib: ^2.5.0
     tsup: ^6.7.0
     turbo: ^1.9.4-canary.9
     typescript: ^5.0.4
+    undici: ^5.22.0
   languageName: unknown
   linkType: soft