Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,16 @@ In our production benchmarks with AWS RDS Postgres and MSK Kafka, Sequin maintai
| HTTP Pull (Native) | ✅ Real-time streaming<br />✅ Backfill existing rows | Consume changes directly from Sequin with exactly-once processing |
| Kafka | ✅ Real-time streaming<br />✅ Backfill existing rows | Stream changes to Apache Kafka topics |
| NATS | ✅ Real-time streaming<br />✅ Backfill existing rows | Stream changes to NATS subjects |
| Postgres | 🛠️ Coming April 2025 | Replicate changes to another Postgres |
| Postgres | 🛠️ Coming June 2025 | Replicate changes to another Postgres |
| RabbitMQ | ✅ Real-time streaming<br />✅ Backfill existing rows | Publish messages to RabbitMQ exchanges |
| Redis | ✅ Real-time streaming<br />✅ Backfill existing rows | `XADD` to Redis Streams |
| Redis Stream | ✅ Real-time streaming<br />✅ Backfill existing rows | `XADD` to Redis Streams |
| Redis String | ✅ Real-time streaming<br />✅ Backfill existing rows | `SET` to Redis keys |
| SQS | ✅ Real-time streaming<br />✅ Backfill existing rows | Send messages to Amazon SQS queues |
| Kinesis | ✅ Real-time streaming<br />✅ Backfill existing rows | Send messages to Amazon Kinesis streams queues |
| Typesense | ✅ Real-time streaming<br />✅ Backfill existing rows | Index database changes with Typesense |
| Elasticsearch | ✅ Real-time streaming<br />✅ Backfill existing rows | Index database changes with Elasticsearch |


| Webhook Subscription (Native) | ✅ Real-time streaming<br />✅ Backfill existing rows | Send changes to any HTTP endpoint |

## Change data capture use cases
Expand Down Expand Up @@ -197,4 +203,4 @@ Sequin uses LiveView + LiveSvelte for its frontend. As a monolith, the entire ap

## Contributing

See [CONTRIBUTING.md](./CONTRIBUTING.md) for detailed instructions on how to contribute to Sequin.
See [CONTRIBUTING.md](./CONTRIBUTING.md) for detailed instructions on how to contribute to Sequin.
8 changes: 8 additions & 0 deletions assets/svelte/consumers/ShowSink.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
HttpPushConsumer,
SqsConsumer,
SnsConsumer,
KinesisConsumer,
RedisStreamConsumer,
KafkaConsumer,
SequinStreamConsumer,
Expand All @@ -31,6 +32,7 @@
import SinkCardHttpPush from "../components/SinkCardHttpPush.svelte";
import SqsSinkCard from "../sinks/sqs/SqsSinkCard.svelte";
import SnsSinkCard from "../sinks/sns/SnsSinkCard.svelte";
import KinesisSinkCard from "../sinks/kinesis/KinesisSinkCard.svelte";
import RedisStreamSinkCard from "../sinks/redis-stream/RedisStreamSinkCard.svelte";
import KafkaSinkCard from "../sinks/kafka/KafkaSinkCard.svelte";
import SequinStreamSinkCard from "../sinks/sequin_stream/SequinStreamSinkCard.svelte";
Expand Down Expand Up @@ -107,6 +109,10 @@
return consumer.sink.type === "sns";
}

function isKinesisConsumer(consumer: Consumer): consumer is KinesisConsumer {
return consumer.sink.type === "kinesis";
}

function isRedisStreamConsumer(
consumer: Consumer,
): consumer is RedisStreamConsumer {
Expand Down Expand Up @@ -1225,6 +1231,8 @@
<SqsSinkCard {consumer} />
{:else if isSnsConsumer(consumer)}
<SnsSinkCard {consumer} />
{:else if isKinesisConsumer(consumer)}
<KinesisSinkCard {consumer} />
{:else if isRedisStreamConsumer(consumer)}
<RedisStreamSinkCard {consumer} />
{:else if isKafkaConsumer(consumer)}
Expand Down
3 changes: 3 additions & 0 deletions assets/svelte/consumers/ShowSinkHeader.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import LinkPushNavigate from "$lib/components/LinkPushNavigate.svelte";
import SqsIcon from "../sinks/sqs/SqsIcon.svelte";
import SnsIcon from "../sinks/sns/SnsIcon.svelte";
import KinesisIcon from "../sinks/kinesis/KinesisIcon.svelte";
import RedisIcon from "../sinks/redis_shared/RedisIcon.svelte";
import KafkaIcon from "../sinks/kafka/KafkaIcon.svelte";
import GcpPubsubIcon from "../sinks/gcp_pubsub/GcpPubsubIcon.svelte";
Expand Down Expand Up @@ -167,6 +168,8 @@
<ElasticsearchIcon class="h-6 w-6 mr-2" />
{:else if consumer.sink.type === "sns"}
<SnsIcon class="h-6 w-6 mr-2" />
{:else if consumer.sink.type === "kinesis"}
<KinesisIcon class="h-6 w-6 mr-2" />
{/if}
<h1 class="text-xl font-semibold">
{consumer.name}
Expand Down
3 changes: 3 additions & 0 deletions assets/svelte/consumers/SinkConsumerForm.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import SinkHttpPushForm from "$lib/consumers/SinkHttpPushForm.svelte";
import SqsSinkForm from "$lib/sinks/sqs/SqsSinkForm.svelte";
import SnsSinkForm from "$lib/sinks/sns/SnsSinkForm.svelte";
import KinesisSinkForm from "$lib/sinks/kinesis/KinesisSinkForm.svelte";
import RedisStreamSinkForm from "$lib/sinks/redis-stream/RedisStreamSinkForm.svelte";
import RedisStringSinkForm from "$lib/sinks/redis-string/RedisStringSinkForm.svelte";
import KafkaSinkForm from "$lib/sinks/kafka/KafkaSinkForm.svelte";
Expand Down Expand Up @@ -671,6 +672,8 @@
<SqsSinkForm errors={errors.consumer} bind:form />
{:else if consumer.type === "sns"}
<SnsSinkForm errors={errors.consumer} bind:form />
{:else if consumer.type === "kinesis"}
<KinesisSinkForm errors={errors.consumer} bind:form />
{:else if consumer.type === "redis_stream"}
<RedisStreamSinkForm errors={errors.consumer} bind:form />
{:else if consumer.type === "redis_string"}
Expand Down
7 changes: 7 additions & 0 deletions assets/svelte/consumers/SinkIndex.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import LinkPatchNavigate from "$lib/components/LinkPatchNavigate.svelte";
import SqsIcon from "../sinks/sqs/SqsIcon.svelte";
import SnsIcon from "../sinks/sns/SnsIcon.svelte";
import KinesisIcon from "../sinks/kinesis/KinesisIcon.svelte";
import RedisIcon from "../sinks/redis_shared/RedisIcon.svelte";
import KafkaIcon from "../sinks/kafka/KafkaIcon.svelte";
import GcpPubsubIcon from "../sinks/gcp_pubsub/GcpPubsubIcon.svelte";
Expand All @@ -44,6 +45,7 @@
| "http_push"
| "sqs"
| "sns"
| "kinesis"
| "redis_stream"
| "kafka"
| "gcp_pubsub"
Expand Down Expand Up @@ -94,6 +96,11 @@
name: "Amazon SNS",
icon: SnsIcon,
},
{
id: "kinesis",
name: "Amazon Kinesis",
icon: KinesisIcon,
},
{
id: "redis_stream",
name: "Redis Stream",
Expand Down
12 changes: 12 additions & 0 deletions assets/svelte/consumers/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export type BaseConsumer = {
| "http_push"
| "sqs"
| "sns"
| "kinesis"
| "redis_stream"
| "kafka"
| "sequin_stream"
Expand Down Expand Up @@ -76,6 +77,16 @@ export type SqsConsumer = BaseConsumer & {
};
};

// Kinesis specific sink
export type KinesisConsumer = BaseConsumer & {
sink: {
type: "kinesis";
stream_arn: string;
access_key_id: string;
secret_access_key: string;
};
};

// Redis specific sink
export type RedisStreamConsumer = BaseConsumer & {
sink: {
Expand Down Expand Up @@ -215,6 +226,7 @@ export type ElasticsearchConsumer = BaseConsumer & {
export type Consumer =
| HttpPushConsumer
| SqsConsumer
| KinesisConsumer
| RedisStreamConsumer
| KafkaConsumer
| SequinStreamConsumer
Expand Down
35 changes: 35 additions & 0 deletions assets/svelte/sinks/kinesis/KinesisIcon.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<svg
viewBox="0 0 80 80"
version="1.1"
xmlns="http://www.w3.org/2000/svg"
xmlns:xlink="http://www.w3.org/1999/xlink"
class={$$props.class}
>
<defs>
<linearGradient x1="0%" y1="100%" x2="100%" y2="0%" id="linearGradient-1">
<stop stop-color="#4D27A8" offset="0%"></stop>
<stop stop-color="#A166FF" offset="100%"></stop>
</linearGradient>
</defs>
<g
id="Icon-Architecture/64/Arch_Amazon-Kinesis-Data-Streams_64"
stroke="none"
stroke-width="1"
fill="none"
fill-rule="evenodd"
>
<g id="Icon-Architecture-BG/64/Analytics" fill="url(#linearGradient-1)">
<rect id="Rectangle" x="0" y="0" width="80" height="80"></rect>
</g>
<g
id="Icon-Service/64/Amazon-Kinesis-Data-Streams"
transform="translate(8.000000, 8.000000)"
fill="#FFFFFF"
>
<path
d="M41,43 L47,43 L47,41 L41,41 L41,43 Z M45,54 L48,54 L48,51 L45,51 L45,54 Z M50,50 L50,55 C50,55.552 49.553,56 49,56 L44,56 C43.448,56 43,55.552 43,55 L43,50 C43,49.448 43.448,49 44,49 L49,49 C49.553,49 50,49.448 50,50 L50,50 Z M44,47 L47,47 L47,45 L44,45 L44,47 Z M51,45 L54,45 L54,42 L51,42 L51,45 Z M55,47 L50,47 C49.448,47 49,46.552 49,46 L49,41 C49,40.448 49.448,40 50,40 L52.975,40 C51.305,38.75 49.242,38 47,38 C42.532,38 38.739,40.947 37.461,45 L42,45 L42,47 L37.051,47 C37.018,47.329 37,47.663 37,48 C37,48.337 37.018,48.671 37.051,49 L41,49 L41,51 L37.461,51 C37.684,51.705 37.989,52.372 38.354,53 L41,53 L41,55 L39.871,55 C41.687,56.849 44.21,58 47,58 C49.79,58 52.314,56.849 54.129,55 L52,55 L52,53 L55.647,53 C56.012,52.372 56.317,51.705 56.539,51 L52,51 L52,49 L56.95,49 C56.983,48.671 57,48.337 57,48 C57,46.447 56.634,44.982 56,43.67 L56,46 C56,46.552 55.553,47 55,47 L55,47 Z M59,48 C59,54.617 53.618,60 47,60 C40.383,60 35,54.617 35,48 C35,41.383 40.383,36 47,36 C53.618,36 59,41.383 59,48 L59,48 Z M40.127,35.817 C25.434,37.238 11,40.72 11,54 L13,54 C13,47.633 16,40.878 37.034,38.185 C37.946,37.258 38.989,36.462 40.127,35.817 L40.127,35.817 Z M17,60 L19,60 C19,54.185 19.012,47.068 33.801,43.377 C34.082,42.577 34.429,41.81 34.844,41.084 C17.014,45.047 17,53.982 17,60 L17,60 Z M7,18 L5,18 C5,25.396 15.039,30.08 34.893,32 C15.039,33.92 5,38.604 5,46 L7,46 C7,41.145 13.756,33 59,33 L59,31 C13.756,31 7,22.855 7,18 L7,18 Z M13,10 L11,10 C11,19.396 16.703,29 59,29 L59,27 C18.465,27 13,18.406 13,10 L13,10 Z M59,23 L59,25 C40.663,25 28.728,22.641 22.51,17.788 C17,13.488 17,8.004 17,4 L19,4 C19,11.566 19,23 59,23 L59,23 Z"
id="Amazon-Kinesis-Data-Streams_Icon_64_Squid"
></path>
</g>
</g>
</svg>
53 changes: 53 additions & 0 deletions assets/svelte/sinks/kinesis/KinesisSinkCard.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
<script lang="ts">
import { ExternalLink } from "lucide-svelte";
import { Card, CardContent } from "$lib/components/ui/card";
import { Button } from "$lib/components/ui/button";
import type { KinesisConsumer } from "../../consumers/types";

export let consumer: KinesisConsumer;

var awsConsoleUrl = "#";

$: {
const arnMatch =
consumer.sink.stream_arn &&
consumer.sink.stream_arn.match(
/^arn:aws:kinesis:([^:]+):[^:]+:stream\/(.+)$/,
);
if (arnMatch) {
const [, region, streamName] = arnMatch;
awsConsoleUrl = `https://${region}.console.aws.amazon.com/kinesis/home?region=${region}#/streams/details/${encodeURIComponent(streamName)}`;
}
}
</script>

<Card>
<CardContent class="p-6">
<div class="flex justify-between items-center mb-4">
<h2 class="text-lg font-semibold">Kinesis Configuration</h2>
<div class="flex space-x-2">
<a href={awsConsoleUrl} target="_blank" rel="noopener noreferrer">
<Button variant="outline" size="sm">
<ExternalLink class="h-4 w-4 mr-2" />
View in AWS Console
</Button>
</a>
</div>
</div>

<div class="space-y-4">
<div class="grid grid-cols-2 gap-4">
<div>
<span class="text-sm text-gray-500">Stream ARN</span>
<div class="mt-2">
<span
class="font-mono bg-slate-50 pl-1 pr-4 py-1 border border-slate-100 rounded-md whitespace-nowrap"
>
{consumer.sink.stream_arn}
</span>
</div>
</div>
</div>
</div>
</CardContent>
</Card>
62 changes: 62 additions & 0 deletions assets/svelte/sinks/kinesis/KinesisSinkForm.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
<script lang="ts">
import { Input } from "$lib/components/ui/input";
import {
Card,
CardContent,
CardHeader,
CardTitle,
} from "$lib/components/ui/card";
import { Label } from "$lib/components/ui/label";

export let form;
export let errors: any = {};
</script>

<Card>
<CardHeader>
<CardTitle>Kinesis Configuration</CardTitle>
</CardHeader>
<CardContent class="flex flex-col gap-4">
<div class="flex flex-col gap-2">
<Label for="stream-arn">Stream ARN</Label>
<Input
id="stream-arn"
bind:value={form.sink.stream_arn}
placeholder="arn:aws:kinesis:..."
/>

{#if errors.sink?.stream_arm}
<p class="text-destructive text-sm">{errors.sink.stream_arn}</p>
{/if}
</div>

<div class="flex flex-col gap-2">
<Label for="access-key">AWS Access Key ID</Label>
<Input
id="access-key"
data-1p-ignore
bind:value={form.sink.access_key_id}
placeholder="Enter your AWS access key ID"
autocomplete="off"
/>
{#if errors.sink?.access_key_id}
<p class="text-destructive text-sm">{errors.sink.access_key_id}</p>
{/if}
</div>

<div class="flex flex-col gap-2">
<Label for="secret-key">AWS Secret Access Key</Label>
<Input
id="secret-key"
data-1p-ignore
type="password"
bind:value={form.sink.secret_access_key}
placeholder="Enter your AWS secret access key"
autocomplete="off"
/>
{#if errors.sink?.secret_access_key}
<p class="text-destructive text-sm">{errors.sink.secret_access_key}</p>
{/if}
</div>
</CardContent>
</Card>
3 changes: 3 additions & 0 deletions docs/docs.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"quickstart/typesense",
"quickstart/elasticsearch",
"quickstart/sqs",
"quickstart/kinesis",
"quickstart/sns",
"quickstart/redis-string",
"quickstart/redis-stream",
Expand Down Expand Up @@ -60,6 +61,7 @@
"how-to/stream-postgres-to-typesense",
"how-to/stream-postgres-to-elasticsearch",
"how-to/stream-postgres-to-sqs",
"how-to/stream-postgres-to-kinesis",
"how-to/stream-postgres-to-sns",
"how-to/stream-postgres-to-redis-string",
"how-to/stream-postgres-to-redis-stream",
Expand Down Expand Up @@ -109,6 +111,7 @@
"reference/sinks/typesense",
"reference/sinks/elasticsearch",
"reference/sinks/sqs",
"reference/sinks/kinesis",
"reference/sinks/sns",
"reference/sinks/kafka",
"reference/sinks/redis-string",
Expand Down
Loading
Loading