Merge pull request #378 from jdno/deploy-sqs

Deploy SQS queue for crates.io to staging

Author: jdno

terragrunt/accounts/crates-io-staging/account.json

@@ -0,0 +1,6 @@
+{
+    "aws": {
+        "profile": "crates-io-staging",
+        "region": "us-east-2"
+    }
+}

terragrunt/accounts/crates-io-staging/crates-io-logs/.terraform.lock.hcl

@@ -0,0 +1,13 @@
+terraform {
+  source = "../../../..//terragrunt/modules/crates-io-logs"
+}
+
+include {
+  path           = find_in_parent_folders()
+  merge_strategy = "deep"
+}
+
+inputs = {
+  bucket_account = 890664054962
+  bucket_arn = "arn:aws:s3:::rust-staging-crates-io-logs"
+}

terragrunt/accounts/crates-io-staging/crates-io-logs/terragrunt.hcl

@@ -0,0 +1,36 @@
+# Infrastructure to Count Crate Downloads
+
+This module creates the infrastructure that enables [crates.io] to count crate
+downloads asynchronously using request logs from our Content Delivery Networks.
+
+Whenever a new archive with request logs is uploaded to S3, S3 pushes an event
+into a SQS queue. [crates.io] monitors the queue and processes incoming events.
+From the event, it can determine what files to fetch from S3, download and then
+parse them, and update the download counts in the database.
+
+```mermaid
+sequenceDiagram
+	static.crates.io ->> S3: Uploads logs
+	S3 ->> SQS: Queues event
+	crates.io ->> SQS: Pulls event from queue
+	crates.io ->> S3: Fetches new log file
+	crates.io ->> crates.io: Parses log file
+	crates.io ->> crates.io: Updates download counts
+```
+
+See [rust-lang/simpleinfra#372] for a detailed discussion of the design.
+
+## AWS Accounts
+
+The infrastructure for [crates.io] has historically been deployed to the
+`legacy` AWS account. For this infrastructure, new accounts have been created
+that follow the new convention of specialized and isolated accounts for
+services.
+
+This requires the S3 bucket with the request logs in the `legacy` account to
+push events into the SQS queue in a different account. And the [crates.io]
+application needs a second set of AWS credentials to pull events from the
+queue.
+
+[crates.io]: https://crates.io
+[rust-lang/simpleinfra#372]: https://github.com/rust-lang/simpleinfra/issues/372

terragrunt/modules/crates-io-logs/README.md

@@ -0,0 +1,20 @@
+terraform {
+  required_version = "~> 1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.32"
+    }
+  }
+}
+
+variable "bucket_account" {
+  type        = number
+  description = "Account ID of the S3 bucket which will send events to the SQS queue"
+}
+
+variable "bucket_arn" {
+  type        = string
+  description = "ARN of the S3 bucket which will send events to the SQS queue"
+}

terragrunt/modules/crates-io-logs/_terraform.tf

@@ -1,16 +1,16 @@
-resource "aws_sqs_queue" "log_event_queue" {
-  name                      = "cdn-log-queue"
+resource "aws_sqs_queue" "cdn_log_event_queue" {
+  name                      = "cdn-log-event-queue"
   receive_wait_time_seconds = 20
 }
 
 resource "aws_sqs_queue_policy" "s3_push" {
-  queue_url = aws_sqs_queue.log_event_queue.id
+  queue_url = aws_sqs_queue.cdn_log_event_queue.id
   policy    = data.aws_iam_policy_document.s3_push_to_queue.json
 }
 
 data "aws_iam_policy_document" "s3_push_to_queue" {
   statement {
-    sid    = "allow-s3-to-push-events"
+    sid    = "AllowS3ToPushEvents"
     effect = "Allow"
     principals {
       type        = "Service"
@@ -19,45 +19,37 @@ data "aws_iam_policy_document" "s3_push_to_queue" {
 
     actions = ["sqs:SendMessage"]
 
-    resources = [aws_sqs_queue.log_event_queue.arn]
+    resources = [aws_sqs_queue.cdn_log_event_queue.arn]
     condition {
       test     = "ArnLike"
       variable = "aws:SourceArn"
-      values   = [data.aws_arn.src_bucket.arn]
+      values   = [var.bucket_arn]
     }
     condition {
       test     = "StringEquals"
       variable = "aws:SourceAccount"
-      values   = [data.aws_arn.src_bucket.account]
+      values   = [var.bucket_account]
     }
   }
 }
 
-data "aws_arn" "src_bucket" {
-  arn = var.src_log_bucket_arn
-}
-
-variable "src_log_bucket_arn" {
-  type        = string
-  description = "Bucket ARN which will send events to the SQS queue"
-}
-
 resource "aws_iam_user" "heroku_access" {
   name = "crates-io-heroku-access"
 }
 
 resource "aws_iam_access_key" "crates_io" {
-  user = aws_iam_user.heroku_access
+  user = aws_iam_user.heroku_access.name
 }
 
-resouce "aws_iam_user_policy" "sqs_read" {
-  name = "heroku-access"
-  user = aws_iam_user.heroku_access.name
+resource "aws_iam_user_policy" "sqs_read" {
+  name   = "heroku-access"
+  user   = aws_iam_user.heroku_access.name
+  policy = data.aws_iam_policy_document.heroku_access.json
 }
 
 data "aws_iam_policy_document" "heroku_access" {
   statement {
-    sid    = "allow-sqs"
+    sid    = "AllowAccessToSQS"
     effect = "Allow"
 
     actions = [
@@ -67,6 +59,6 @@ data "aws_iam_policy_document" "heroku_access" {
       "sqs:ReceiveMessage",
     ]
 
-    resources = [aws_sqs_queue.log_event_queue.arn]
+    resources = [aws_sqs_queue.cdn_log_event_queue.arn]
   }
 }