Support build image in private networks by retrieving dependencies from s3 bucket (aws#6296)

Author: hgreebe

CHANGELOG.md

@@ -19,6 +19,7 @@ CHANGELOG
 
 **CHANGES**
 - Upgrade Cinc Client to version to 18.4.12 from 18.2.7.
+- Allow build-image to be run in an isolated network.
 
 3.9.3
 ------

cli/src/pcluster/constants.py

@@ -294,16 +294,7 @@ class Operation(Enum):
     VERSION = "version"
 
 
-UNSUPPORTED_OPERATIONS_MAP = {
-    Operation.BUILD_IMAGE: ["us-iso"],
-    Operation.DELETE_IMAGE: ["us-iso"],
-    Operation.DESCRIBE_IMAGE: ["us-iso"],
-    Operation.LIST_IMAGES: ["us-iso"],
-    Operation.EXPORT_IMAGE_LOGS: ["us-iso"],
-    Operation.GET_IMAGE_LOG_EVENTS: ["us-iso"],
-    Operation.GET_IMAGE_STACK_EVENTS: ["us-iso"],
-    Operation.LIST_IMAGE_LOG_STREAMS: ["us-iso"],
-}
+UNSUPPORTED_OPERATIONS_MAP = {}
 
 MAX_TAGS_COUNT = 40  # Tags are limited to 50, reserve some tags for parallelcluster specified tags
 

cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml

@@ -222,10 +222,16 @@ phases:
               curl --retry 3 -L {{ build.CincUrl.outputs.stdout }} | bash -s -- -v {{ ChefVersion }}
 
               if [[ -e ${!CA_CERTS_FILE} ]]; then
+                mkdir -p /opt/cinc/embedded/ssl/certs
                 ln -sf ${!CA_CERTS_FILE} /opt/cinc/embedded/ssl/certs/cacert.pem
               fi
+              
+              curl --retry 3 -L -o gems.tgz https://${AWS::Region}-aws-parallelcluster.s3.${AWS::Region}.amazonaws.com/archives/dependencies/ruby/gems.tgz
+              tar -xf gems.tgz
+              
+              cd vendor/cache
 
-              /opt/cinc/embedded/bin/gem install --no-document berkshelf:{{ BerkshelfVersion }}
+              /opt/cinc/embedded/bin/gem install --local --no-document berkshelf:{{ BerkshelfVersion }}
 
       # Download and vendor Cookbook
       - name: DownloadCookbook
@@ -235,9 +241,10 @@ phases:
             - |
               set -v
               mkdir -p /etc/chef && sudo chown -R root:root /etc/chef
+              
               curl --retry 3 -L -o /etc/chef/aws-parallelcluster-cookbook.tgz "{{ build.CookbookUrl.outputs.stdout }}"
 
-              mkdir /tmp/cookbooks
+              mkdir -p /tmp/cookbooks
               cd /tmp/cookbooks
               tar -xzf /etc/chef/aws-parallelcluster-cookbook.tgz
 

tests/integration-tests/configs/build_image_iso.yaml

@@ -0,0 +1,22 @@
+{%- import 'common.jinja2' as common with context -%}
+{% if REGIONS  %}
+{%- set REGIONS = [ REGIONS ] -%}
+{% else %}
+{%- set REGIONS = ["us-isob-east-1","us-iso-east-1"] -%}
+{% endif %}
+{%- set INSTANCES = ["c5.xlarge"] -%}
+{% if OSS  %}
+{%- set OSS = [ OSS ] -%}
+{% else %}
+{%- set OSS = ["alinux2"] -%}
+{% endif %}
+{%- set SCHEDULERS = ["slurm"] -%}
+---
+test-suites:
+  createami:
+    test_createami.py::test_build_image:
+      dimensions:
+        - regions: {{ REGIONS }}
+          instances: {{ INSTANCES }}
+          schedulers: {{ SCHEDULERS }}
+          oss: {{ OSS }}