[aws] Initial commit of AWS infrastructure

batch/batch/cloud/utils.py

@@ -2,6 +2,7 @@
 from typing import Any, Dict, Set
 
 from gear.cloud_config import get_azure_config, get_gcp_config
+from hailtop.aiocloud.aioaws import S3AsyncFS
 from hailtop.aiocloud.aioazure import AzureAsyncFS
 from hailtop.aiocloud.aiogoogle import GoogleStorageAsyncFS
 
@@ -41,6 +42,8 @@ def _query_storage_url_prefix(subfolder_envvar: str) -> str:
 
     if cloud == 'gcp':
         assert GoogleStorageAsyncFS.valid_url(url_prefix)
+    elif cloud == 'aws':
+        assert S3AsyncFS.valid_url(url_prefix)
     else:
         assert cloud == 'azure'
         assert AzureAsyncFS.valid_url(url_prefix)

infra/aws/README.md

@@ -0,0 +1,27 @@
+# Hail on AWS
+
+This document is a WIP as we build out Hail infrastructure on AWS.
+
+## Prerequisites:
+
+- Install the AWS CLI and pass it credentials using `aws configure` or `aws login`.
+- Set the default region for the AWS CLI using `aws configure`. The Hail backend is deployed in `us-east-1.`
+- Check out the Hail repository and switch to the `$HAIL/infra/aws` directory.
+- Create a new directory for your instance of Hail. Copy `hail-is/parameters.json` into this directory and edit the parameter values to match the desired configuration for your cluster.
+- Create an IAM role for the CloudFormation stack. CloudFormation will use this role to manage all infrastructure:
+```
+export STACK_ROLE_NAME=hail-cloudformation-role
+export STACK_ROLE_ARN=$(aws iam create-role --role-name $STACK_ROLE_NAME --assume-role-policy-document "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"cloudformation.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" --query "Role.Arn" --output text)
+aws iam attach-role-policy --role-name $STACK_ROLE_NAME --policy-arn arn:aws:iam::aws:policy/AdministratorAccess
+```
+
+## Creating the Cluster
+
+Create the cluster infrastructure using CloudFormation:
+
+```
+export INSTANCE_NAME=<name for this instance of Hail>
+aws cloudformation create-stack --stack-name hail-vdc --template-body file://main.yaml --role-arn $STACK_ROLE_ARN --parameters file://${INSTANCE_NAME}/parameters.json --capabilities CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND --disable-rollback
+```
+
+The stack will take 15 to 20 minutes to finish creating.

infra/aws/hail-is/parameters.json

@@ -0,0 +1,30 @@
+[
+  {
+    "ParameterKey": "VpcCidrBlock",
+    "ParameterValue": "172.31.0.0/16"
+  },
+  {
+    "ParameterKey": "ClusterServiceCidrBlock",
+    "ParameterValue": "172.20.0.0/16"
+  },
+  {
+    "ParameterKey": "SubnetCidrBlockA",
+    "ParameterValue": "172.31.0.0/20"
+  },
+  {
+    "ParameterKey": "SubnetCidrBlockB",
+    "ParameterValue": "172.31.16.0/20"
+  },
+  {
+    "ParameterKey": "SubnetCidrBlockC",
+    "ParameterValue": "172.31.32.0/20"
+  },
+  {
+    "ParameterKey": "PublicAccessIpList",
+    "ParameterValue": "69.173.64.0/18, 69.173.64.0/19, 69.173.96.0/19, 69.173.95.0/24, 69.173.97.0/24"
+  },
+  {
+    "ParameterKey": "DomainName",
+    "ParameterValue": "aws.hail.is"
+  }
+]