diff --git a/CHANGELOG.md b/CHANGELOG.md index da3e19a..3d398d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2022-08-09 + +### Changed + +- README.md file to include correct instructions on how to use the ECS executor. + ## [0.3.0] - 2022-08-09 ### Added diff --git a/README.md b/README.md index 032fa55..ebce69a 100644 --- a/README.md +++ b/README.md @@ -2,45 +2,41 @@
-![covalent logo](https://github.com/AgnostiqHQ/covalent/blob/master/doc/source/_static/dark.png#gh-dark-mode-only) -![covalent logo](https://github.com/AgnostiqHQ/covalent/blob/master/doc/source/_static/light.png#gh-light-mode-only) - -  +
-## Covalent Fargate Plugin +## Covalent ECS Plugin -Covalent is a Pythonic workflow tool used to execute tasks on advanced computing hardware. This executor plugin interfaces Covalent with AWS Fargate via the [Elastic Container Service](https://docs.aws.amazon.com/ecs/index.html). In order for workflows to be deployable, users must have AWS credentials attached to the [CovalentFargateExecutorPolicy](https://github.com/AgnostiqHQ/covalent-fargate-executor/infra/iam/CovalentFargateExecutorPolicy.json). Users will need additional permissions to provision or manage cloud infrastructure used by this plugin. +Covalent is a Pythonic workflow tool used to execute tasks on advanced computing hardware. This executor plugin interfaces Covalent with AWS [Elastic Container Service](https://docs.aws.amazon.com/ecs/index.html) where the tasks are run using Fargate. In order for workflows to be deployable, users must have AWS credentials attached to the [CovalentECSExecutorPolicy](https://github.com/AgnostiqHQ/covalent-ecs-plugin/blob/main/infra/iam/CovalentECSExecutorPolicy.json). Users will need additional permissions to provision or manage cloud infrastructure used by this plugin. To use this plugin with Covalent, clone this repository and install it using `pip`: ``` -git clone git@github.com:AgnostiqHQ/covalent-fargate-plugin.git -cd covalent-fargate-plugin +git clone git@github.com:AgnostiqHQ/covalent-ecs-plugin.git +cd covalent-ecs-plugin pip install . ``` -Users must add the correct entries to their Covalent [configuration](https://covalent.readthedocs.io/en/latest/how_to/config/customization.html) to support the Fargate plugin. Below is an example which works using some basic infrastructure created for testing purposes: +Users must add the correct entries to their Covalent [configuration](https://covalent.readthedocs.io/en/latest/how_to/config/customization.html) to support the ECS plugin. Below is an example which works using some basic infrastructure created for testing purposes: ```console -[executors.fargate] +[executors.ecs] credentials = "/home/user/.aws/credentials" -profile = "" +profile = "default" s3_bucket_name = "covalent-fargate-task-resources" ecr_repo_name = "covalent-fargate-task-images" ecs_cluster_name = "covalent-fargate-cluster" ecs_task_family_name = "covalent-fargate-tasks" +ecs_task_execution_role_name = "ecsTaskExecutionRole" ecs_task_role_name = "CovalentFargateTaskRole" +ecs_task_subnet_id = "subnet-871545e1" +ecs_task_security_group_id = "sg-0043541a" +ecs_task_log_group_name = "covalent-fargate-task-logs" vcpu = 0.25 memory = 0.5 cache_dir = "/tmp/covalent" -ecs_task_subnets = "subnet-994c4697,subnet-861e43d9,subnet-779cc356,subnet-326a0e03,subnet-871545e1,subnet-6793732b" -ecs_task_vpc = "vpc-b2bdd0cf" -ecs_task_security_groups = "sg-0043541a" poll_freq = 10 -ecs_task_execution_role_name = "ecsTaskExecutionRole" -ecs_task_log_group_name = "covalent-fargate-task-logs" ``` Within a workflow, users can then decorate electrons using these default settings: @@ -48,7 +44,7 @@ Within a workflow, users can then decorate electrons using these default setting ```python import covalent as ct -@ct.electron(executor="fargate") +@ct.electron(executor="ecs") def my_task(x, y): return x + y ``` @@ -56,9 +52,11 @@ def my_task(x, y): or use a class object to customize the resources and other behavior: ```python -executor = ct.executor.FargateExecutor( +executor = ct.executor.ECSExecutor( vcpu=1, - memory=2 + memory=2, + ecs_task_subnet_id="subnet-871545e1", + ecs_task_security_group_id="sg-0043541a" ) @ct.electron(executor=executor) @@ -66,11 +64,13 @@ def my_custom_task(x, y): return x + y ``` +Ensure that Docker is running on the client side machine before deploying the workflow. + For more information about how to get started with Covalent, check out the project [homepage](https://github.com/AgnostiqHQ/covalent) and the official [documentation](https://covalent.readthedocs.io/en/latest/). ## Release Notes -Release notes are available in the [Changelog](https://github.com/AgnostiqHQ/covalent-fargate-executor/blob/main/CHANGELOG.md). +Release notes are available in the [Changelog](https://github.com/AgnostiqHQ/covalent-ecs-executor/blob/main/CHANGELOG.md). ## Citation @@ -81,4 +81,4 @@ Please use the following citation in any publications: ## License -Covalent is licensed under the GNU Affero GPL 3.0 License. Covalent may be distributed under other licenses upon request. See the [LICENSE](https://github.com/AgnostiqHQ/covalent-fargate-executor/blob/main/LICENSE) file or contact the [support team](mailto:support@agnostiq.ai) for more details. +Covalent is licensed under the GNU Affero GPL 3.0 License. Covalent may be distributed under other licenses upon request. See the [LICENSE](https://github.com/AgnostiqHQ/covalent-ecs-executor/blob/main/LICENSE) file or contact the [support team](mailto:support@agnostiq.ai) for more details. diff --git a/covalent_ecs_plugin/ecs.py b/covalent_ecs_plugin/ecs.py index 6443d46..0a9c6af 100644 --- a/covalent_ecs_plugin/ecs.py +++ b/covalent_ecs_plugin/ecs.py @@ -20,71 +20,6 @@ """AWS ECSExecutor plugin for the Covalent dispatcher.""" -# Infrastructure required for this executor: -# 1. VPC -# - IPv4 CIDR: 10.0.0.0/16 -# 2. Private Subnets (1 per zone) -# - IPv4 CIDR: 10.0.X.0/24 -# 3. Route Table -# 4. Elastic IP -# 5. NAT Gateway -# 6. Outbound route to NAT Gateway -# - 0.0.0.0/0 -> NAT Addr -# 7. Security Group (empty) -# 8. S3 Bucket -# 9. ECR Repository -# - Immutable tags -# - KMS encryption -# 10. ECS Cluster -# 11. CloudWatch Log Group -# 12. IAM Policy - CovalentFargateTaskExecutionPolicy (see below) -# 13. IAM Role - CovalentFargateTaskExecutionRole -# 14. IAM Policy - CovalentFargateTaskPolicy (see below) -# 15. IAM Role - CovalentFargateTaskRole -# 16. IAM Policy - CovalentFargateExecutorPolicy (see below) -# 17. IAM Policy - CovalentFargateExecutorInfraPolicy (see below) -# 18. ECS Task Definition - created at runtime -# 19. ECS Task - created at runtime - - -# IAM policies needed for the actions related to this executor: -# 1. CovalentFargateExecutorPolicy: the policy needed to use the FargateExecutor, without -# provisioning infrastructure -- see infra/iam/CovalentFargateExecutorPolicy.json -# 2. CovalentFargateExecutorInfraPolicy: Same as above, except additionally allowing provisioning; -# Below is an in-progress list. -# - Action: -# - logs:CreateLogGroup -# - ecs:CreateCluster -# - ecr:CreateRepository -# - s3:CreateBucket -# Resource: * -# 3. CovalentFargateTaskExecutionPolicy: ECS task execution role's policy -- see -# infra/iam/CovalentFargateTaskExecutionPolicy.json -# 4. CovalentFargateTaskPolicy: ECS task's policy -- see infra/iam/CovalentFargateTaskPolicy.json - - -# Network configuration: -# 1. There are new changes in Fargate 1.4.0 which require the ECS agent to be able to communicate -# to the internet in order to access ECR images. This means that either we use public subnets -# in a VPC connected to an internet gateway, or we can use private subnets which route -# 0.0.0.0/0 to one or more NAT gateways. If we choose the former option, it is important to -# provide "assignPublicIp": "ENABLED" in the network configuration when calling ecs.run_task. -# 2. For the purposes of testing this executor, the default VPC and default subnets are used in -# us-east-1. These fall into the first category above. -# 3. The recommended option for production is to use a set of private subnets all connected -# to the same NAT gateway. This also will need a dedicated VPC. - - -# Synchronization: -# 1. Consider adding a sync/async bool option to execute. Sync should poll the result, while -# async should include a callback within the script that's run on the remote machine. This will -# allow the runner to interact with any given executor in both ways; we expect synchronous behavior -# in a self-hosted runner, and async behavior in the hosted (Covalent Cloud) runner. -# 2. The lifecycle of an ECS task includes [Provisioning -> Pending -> Activating -> Running -> -# Deactivating -> Stopping -> Deprovisioning -> Stopped]. Polling means waiting until the Stopped -# state has been reached, then returning the - - import base64 import os import re @@ -312,8 +247,8 @@ def execute( count=1, networkConfiguration={ "awsvpcConfiguration": { - "subnets": self.ecs_task_subnet_id.split(","), - "securityGroups": self.ecs_task_security_group_id.split(","), + "subnets": [self.ecs_task_subnet_id], + "securityGroups": [self.ecs_task_security_group_id], # This is only needed if we're using public subnets "assignPublicIp": "ENABLED", }, diff --git a/doc/source/_static/aws_ecs_readme_banner.jpg b/doc/source/_static/aws_ecs_readme_banner.jpg new file mode 100644 index 0000000..2ecc024 Binary files /dev/null and b/doc/source/_static/aws_ecs_readme_banner.jpg differ diff --git a/infra/iam/CovalentFargateExecutorPolicy.json b/infra/iam/CovalentECSExecutorPolicy.json similarity index 55% rename from infra/iam/CovalentFargateExecutorPolicy.json rename to infra/iam/CovalentECSExecutorPolicy.json index 1d1fd57..98a0235 100644 --- a/infra/iam/CovalentFargateExecutorPolicy.json +++ b/infra/iam/CovalentECSExecutorPolicy.json @@ -2,7 +2,7 @@ "Version": "2012-10-17", "Statement": [ { - "Sid": "ECSTaskMgmt", + "Sid": "VisualEditor0", "Effect": "Allow", "Action": [ "ecs:RunTask", @@ -17,71 +17,40 @@ } }, { - "Sid": "ECSTaskRegistration", - "Effect": "Allow", - "Action": [ - "ecs:RegisterTaskDefinition" - ], - "Resource": "*" - }, - { - "Sid": "ECRAuth", + "Sid": "VisualEditor1", "Effect": "Allow", "Action": [ + "ecs:RegisterTaskDefinition", "ecr:GetAuthorizationToken" ], "Resource": "*" }, { - "Sid": "ECRUpload", + "Sid": "VisualEditor2", "Effect": "Allow", "Action": [ + "iam:GetRole", "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "ecr:BatchCheckLayerAvailability", - "ecr:InitiateLayerUpload", + "s3:ListBucket", "ecr:UploadLayerPart", + "ecr:PutImage", + "s3:PutObject", + "s3:GetObject", + "iam:PassRole", + "ecr:BatchGetImage", "ecr:CompleteLayerUpload", - "ecr:PutImage" - ], - "Resource": [ - "arn:aws:ecr:::repository/" - ] - }, - { - "Sid": "IAMRoles", - "Effect": "Allow", - "Action": [ - "iam:GetRole", - "iam:PassRole" + "logs:GetLogEvents", + "ecr:InitiateLayerUpload", + "ecr:BatchCheckLayerAvailability" ], "Resource": [ + "arn:aws:ecr:::repository/", "arn:aws:iam:::role/CovalentFargateTaskRole", - "arn:aws:iam:::role/ecsTaskExecutionRole" - ] - }, - { - "Sid": "ObjectStore", - "Effect": "Allow", - "Action": [ - "s3:ListBucket", - "s3:PutObject", - "s3:GetObject" - ], - "Resource": [ + "arn:aws:iam:::role/ecsTaskExecutionRole", + "arn:aws:logs:*::log-group::log-stream:*", "arn:aws:s3:::/*", "arn:aws:s3:::" ] - }, - { - "Sid": "LogRead", - "Effect": "Allow", - "Action": [ - "logs:GetLogEvents" - ], - "Resource": [ - "arn:aws:logs:::log-group::log-stream:*" - ] } ] } diff --git a/infra/iam/CovalentFargateTaskExecutionPolicy.json b/infra/iam/CovalentFargateTaskExecutionPolicy.json deleted file mode 100644 index 547590a..0000000 --- a/infra/iam/CovalentFargateTaskExecutionPolicy.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "ECSAgent", - "Effect": "Allow", - "Action": [ - "ecr:GetAuthorizationToken", - "ecr:BatchCheckLayerAvailability", - "ecr:GetDownloadUrlForLayer", - "ecr:BatchGetImage", - "logs:CreateLogStream", - "logs:PutLogEvents" - ], - "Resource": "*" - } - ] -} diff --git a/infra/iam/CovalentFargateTaskPolicy.json b/infra/iam/CovalentFargateTaskPolicy.json deleted file mode 100644 index 320fad0..0000000 --- a/infra/iam/CovalentFargateTaskPolicy.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "BraketAccess", - "Effect": "Allow", - "Action": "braket:*", - "Resource": "*" - }, - { - "Sid": "ObjectStore", - "Effect": "Allow", - "Action": [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucket" - ], - "Resource": [ - "arn:aws:s3:::/*", - "arn:aws:s3:::" - ] - } - ] -}