diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py index e042fe85c..d8eacf655 100644 --- a/src/_nebari/stages/infrastructure/__init__.py +++ b/src/_nebari/stages/infrastructure/__init__.py @@ -168,7 +168,7 @@ class AWSInputVars(schema.Base): kubernetes_version: str eks_endpoint_access: Optional[ Literal["private", "public", "public_and_private"] - ] = "public" + ] = "public_and_private" eks_kms_arn: Optional[str] = None eks_public_access_cidrs: Optional[List[str]] = ["0.0.0.0/0"] node_groups: List[AWSNodeGroupInputVars] @@ -457,7 +457,7 @@ class AmazonWebServicesProvider(schema.Base): node_groups: Dict[str, AWSNodeGroup] = DEFAULT_AWS_NODE_GROUPS eks_endpoint_access: Optional[ Literal["private", "public", "public_and_private"] - ] = "public" + ] = "public_and_private" eks_public_access_cidrs: Optional[List[str]] = ["0.0.0.0/0"] eks_kms_arn: Optional[str] = None existing_subnet_ids: Optional[List[str]] = None diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf index ec0cbb660..a7b50279e 100644 --- a/src/_nebari/stages/infrastructure/template/aws/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/main.tf @@ -9,10 +9,10 @@ data "aws_partition" "current" {} locals { # Only override_network if both existing_subnet_ids and existing_security_group_id are not null. - override_network = (var.existing_subnet_ids != null) && (var.existing_security_group_id != null) - subnet_ids = local.override_network ? var.existing_subnet_ids : module.network[0].subnet_ids - security_group_id = local.override_network ? var.existing_security_group_id : module.network[0].security_group_id - partition = data.aws_partition.current.partition + override_network = (var.existing_subnet_ids != null) && (var.existing_security_group_id != null) + private_subnet_ids = local.override_network ? var.existing_subnet_ids : module.network[0].private_subnet_ids + security_group_id = local.override_network ? var.existing_security_group_id : module.network[0].security_group_id + partition = data.aws_partition.current.partition } # ==================== ACCOUNTING ====================== @@ -50,6 +50,7 @@ module "network" { vpc_cidr_block = var.vpc_cidr_block aws_availability_zones = length(var.availability_zones) >= 2 ? var.availability_zones : slice(sort(data.aws_availability_zones.awszones.names), 0, 2) + region = var.region } @@ -70,7 +71,7 @@ module "efs" { name = "${local.cluster_name}-jupyterhub-shared" tags = local.additional_tags - efs_subnets = local.subnet_ids + efs_subnets = local.private_subnet_ids efs_security_groups = [local.security_group_id] } @@ -88,7 +89,7 @@ module "kubernetes" { region = var.region kubernetes_version = var.kubernetes_version - cluster_subnets = local.subnet_ids + cluster_subnets = local.private_subnet_ids cluster_security_groups = [local.security_group_id] node_group_additional_policies = [ diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf index aea7c9009..e67e62a14 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/network/main.tf @@ -7,15 +7,36 @@ resource "aws_vpc" "main" { tags = merge({ Name = var.name }, var.tags, var.vpc_tags) } -resource "aws_subnet" "main" { +resource "aws_subnet" "public" { count = length(var.aws_availability_zones) - availability_zone = var.aws_availability_zones[count.index] - cidr_block = cidrsubnet(var.vpc_cidr_block, var.vpc_cidr_newbits, count.index) - vpc_id = aws_vpc.main.id - map_public_ip_on_launch = true + availability_zone = var.aws_availability_zones[count.index] + cidr_block = cidrsubnet(var.vpc_cidr_block, var.vpc_cidr_newbits, count.index) + vpc_id = aws_vpc.main.id - tags = merge({ Name = "${var.name}-subnet-${count.index}" }, var.tags, var.subnet_tags) + tags = merge({ Name = "${var.name}-public-subnet-${count.index}", "kubernetes.io/role/elb" = 1 }, var.tags, var.subnet_tags) + + lifecycle { + ignore_changes = [ + availability_zone + ] + } +} + +moved { + from = aws_subnet.main + to = aws_subnet.public +} + + +resource "aws_subnet" "private" { + count = length(var.aws_availability_zones) + + availability_zone = var.aws_availability_zones[count.index] + cidr_block = cidrsubnet(var.vpc_cidr_block, var.vpc_cidr_newbits, count.index + length(var.aws_availability_zones)) + vpc_id = aws_vpc.main.id + + tags = merge({ Name = "${var.name}-private-subnet-${count.index}" }, var.tags, var.subnet_tags) lifecycle { ignore_changes = [ @@ -30,7 +51,25 @@ resource "aws_internet_gateway" "main" { tags = merge({ Name = var.name }, var.tags) } -resource "aws_route_table" "main" { +resource "aws_eip" "nat-gateway-eip" { + count = length(var.aws_availability_zones) + + domain = "vpc" + + tags = merge({ Name = "${var.name}-nat-gateway-eip-${count.index}" }, var.tags) +} + +resource "aws_nat_gateway" "main" { + count = length(var.aws_availability_zones) + + allocation_id = aws_eip.nat-gateway-eip[count.index].id + subnet_id = aws_subnet.public[count.index].id + + tags = merge({ Name = "${var.name}-nat-gateway-${count.index}" }, var.tags) + depends_on = [aws_internet_gateway.main] +} + +resource "aws_route_table" "public" { vpc_id = aws_vpc.main.id route { @@ -41,11 +80,36 @@ resource "aws_route_table" "main" { tags = merge({ Name = var.name }, var.tags) } -resource "aws_route_table_association" "main" { +moved { + from = aws_route_table.main + to = aws_route_table.public +} + +resource "aws_route_table" "private" { count = length(var.aws_availability_zones) - subnet_id = aws_subnet.main[count.index].id - route_table_id = aws_route_table.main.id + vpc_id = aws_vpc.main.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_nat_gateway.main[count.index].id + } + + tags = merge({ Name = var.name }, var.tags) +} + +resource "aws_route_table_association" "public" { + count = length(var.aws_availability_zones) + + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +resource "aws_route_table_association" "private" { + count = length(var.aws_availability_zones) + + subnet_id = aws_subnet.private[count.index].id + route_table_id = aws_route_table.private[count.index].id } resource "aws_security_group" "main" { @@ -62,7 +126,6 @@ resource "aws_security_group" "main" { cidr_blocks = [var.vpc_cidr_block] } - #trivy:ignore:AVD-AWS-0104 egress { description = "Allow all ports and protocols to exit the security group" from_port = 0 @@ -73,3 +136,61 @@ resource "aws_security_group" "main" { tags = merge({ Name = var.name }, var.tags, var.security_group_tags) } + +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.s3" + vpc_endpoint_type = "Gateway" + route_table_ids = aws_route_table.private[*].id + tags = merge({ Name = "${var.name}-s3-endpoint" }, var.tags) +} + +resource "aws_vpc_endpoint" "ecr_api" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.ecr.api" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + security_group_ids = [aws_security_group.main.id] + subnet_ids = aws_subnet.private[*].id + tags = merge({ Name = "${var.name}-ecr-api-endpoint" }, var.tags) +} + +resource "aws_vpc_endpoint" "ecr_dkr" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.ecr.dkr" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + security_group_ids = [aws_security_group.main.id] + subnet_ids = aws_subnet.private[*].id + tags = merge({ Name = "${var.name}-ecr-dkr-endpoint" }, var.tags) +} + +resource "aws_vpc_endpoint" "elasticloadbalancing" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.elasticloadbalancing" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + security_group_ids = [aws_security_group.main.id] + subnet_ids = aws_subnet.private[*].id + tags = merge({ Name = "${var.name}-elb-endpoint" }, var.tags) +} + +resource "aws_vpc_endpoint" "sts" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.sts" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + security_group_ids = [aws_security_group.main.id] + subnet_ids = aws_subnet.private[*].id + tags = merge({ Name = "${var.name}-sts-endpoint" }, var.tags) +} + +resource "aws_vpc_endpoint" "eks" { + vpc_id = aws_vpc.main.id + service_name = "com.amazonaws.${var.region}.eks" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + security_group_ids = [aws_security_group.main.id] + subnet_ids = aws_subnet.private[*].id + tags = merge({ Name = "${var.name}-eks-endpoint" }, var.tags) +} diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf index 107e1ce47..a29991e18 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/network/outputs.tf @@ -3,9 +3,14 @@ output "security_group_id" { value = aws_security_group.main.id } -output "subnet_ids" { - description = "AWS VPC subnet ids" - value = aws_subnet.main[*].id +output "public_subnet_ids" { + description = "AWS VPC public subnet ids" + value = aws_subnet.public[*].id +} + +output "private_subnet_ids" { + description = "AWS VPC private subnet ids" + value = aws_subnet.private[*].id } output "vpc_id" { diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf b/src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf index 11e22e023..d484facf6 100644 --- a/src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf +++ b/src/_nebari/stages/infrastructure/template/aws/modules/network/variables.tf @@ -40,5 +40,11 @@ variable "vpc_cidr_block" { variable "vpc_cidr_newbits" { description = "VPC cidr number of bits to support 2^N subnets" type = number - default = 2 + default = 2 # allows 4 /18 subnets with 16382 addresses each +} + +variable "region" { + description = "AWS region to operate infrastructure" + type = string + } diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py index fa917c016..eac24ea57 100644 --- a/src/_nebari/stages/terraform_state/__init__.py +++ b/src/_nebari/stages/terraform_state/__init__.py @@ -248,7 +248,8 @@ def check_immutable_fields(self): # Return a default (mutable) extra field schema if bottom level is not a Pydantic model (such as a free-form 'overrides' block) if isinstance(bottom_level_schema, BaseModel): extra_field_schema = schema.ExtraFieldSchema( - **bottom_level_schema.model_fields[keys[-1]].json_schema_extra or {} + **type(bottom_level_schema).model_fields[keys[-1]].json_schema_extra + or {} ) else: extra_field_schema = schema.ExtraFieldSchema()