Skip to content

Commit a3d8a4a

Browse files
hgreebeHimani Anil Deshpande
hgreebe
and
Himani Anil Deshpande
committed
[Isolated] Update Pypi dependencies and install efs-proxy dependency (#3011)
* [Isolated] Install cfn-dependencies only for AL2 * Revert "[Isolated] Install cfn-dependencies only for AL2" This reverts commit 976d479. * [Isolated] USe latest cfn-dependencies * [Isolated] Using Git REF for Uploading cookbook * [Isolated] Rename the cfn-dependecies files * [Isolated] Chnage the name of Cookbook Dependencies and the folder name inside the Tar * [Isolated] Chnage the name of CFN Dependencies and the folder name inside the Tar * [Isolated] Installing Cfn-bootstrap using `--no-build-isolation` as 3.12.8 uses setup.py based installation where it uses a isolated build instead of looking at existing site-packages * [Isolated] Install efs-proxy cargo dependecies for isolated environment * [Isolated] Install new node pypi dependencies and move efs-proxy installation * [Isolated] Only install efs-proxy deps when in adc regions * [Isolated] Only install efs-proxy-deps in adc * [Isolated] Fix unit tests * [Isolated] Test python pacakges are installed when in an ADC region --------- Co-authored-by: Himani Anil Deshpande <[email protected]>
1 parent e6ed15f commit a3d8a4a

File tree

7 files changed

+138
-41
lines changed

7 files changed

+138
-41
lines changed

‎cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,22 @@
1919

2020
# TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
2121
# bash block to a recipe that uses the pyenv resource.
22-
if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2"
22+
command = if aws_region.start_with?("us-iso")
23+
"pip install . --no-build-isolation"
24+
else
25+
"pip install ."
26+
end
27+
28+
if aws_region.start_with?("us-iso")
29+
dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
30+
dependency_folder_name = dependency_package_name
31+
if platform?('amazon') && node['platform_version'] == "2"
32+
dependency_package_name = "node-dependencies"
33+
dependency_folder_name = "node"
34+
end
35+
2336
remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
24-
source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
37+
source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
2538
mode '0644'
2639
retries 3
2740
retry_delay 5
@@ -35,7 +48,7 @@
3548
code <<-REQ
3649
set -e
3750
tar xzf node-dependencies.tgz
38-
cd node
51+
cd #{dependency_folder_name}
3952
#{node_virtualenv_path}/bin/pip install * -f ./ --no-index
4053
REQ
4154
end
@@ -59,7 +72,7 @@
5972
mkdir aws-parallelcluster-custom-node
6073
tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node
6174
cd aws-parallelcluster-custom-node/*aws-parallelcluster-node*
62-
pip install .
75+
#{command}
6376
deactivate
6477
NODE
6578
end

‎cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,14 @@
3434
end
3535

3636
if aws_region.start_with?("us-iso")
37+
dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
38+
dependency_folder_name = dependency_package_name
39+
if platform?('amazon') && node['platform_version'] == "2"
40+
dependency_package_name = "cfn-dependencies"
41+
dependency_folder_name = "cfn"
42+
end
3743
remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
38-
source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
44+
source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz"
3945
mode '0644'
4046
retries 3
4147
retry_delay 5
@@ -49,7 +55,7 @@
4955
code <<-REQ
5056
set -e
5157
tar xzf cfn-dependencies.tgz
52-
cd cfn
58+
cd #{dependency_folder_name}
5359
#{virtualenv_path}/bin/pip install * -f ./ --no-index
5460
REQ
5561
end
@@ -73,11 +79,16 @@
7379
retry_delay 5
7480
end
7581

82+
command = if aws_region.start_with?("us-iso")
83+
"#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation"
84+
else
85+
"#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}"
86+
end
7687
bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do
7788
user 'root'
7889
group 'root'
7990
cwd '/tmp'
80-
code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}"
91+
code command
8192
creates "#{virtualenv_path}/bin/cfn-hup"
8293
end
8394

‎cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,55 @@
2121
use 'partial/_redhat_based'
2222
use 'partial/_install_from_tar'
2323
use 'partial/_mount_umount'
24+
25+
def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version)
26+
<<-EFSUTILSINSTALL
27+
set -e
28+
tar xf #{efs_utils_tarball}
29+
mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/
30+
cd efs-utils-#{efs_utils_version}/src/proxy/
31+
tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz
32+
cargo build --offline
33+
cd ../..
34+
make rpm
35+
yum -y install ./build/#{efs_utils_package}*rpm
36+
EFSUTILSINSTALL
37+
end
38+
39+
def prerequisites
40+
%w(rpm-build make rust cargo openssl-devel)
41+
end
42+
43+
action :install_efs_utils do
44+
package_name = "amazon-efs-utils"
45+
package_version = new_resource.efs_utils_version
46+
efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
47+
48+
if aws_region.start_with?("us-iso")
49+
50+
efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz"
51+
efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}"
52+
efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}"
53+
remote_file efs_proxy_deps_tarball do
54+
source efs_proxy_deps_url
55+
mode '0644'
56+
retries 3
57+
retry_delay 5
58+
action :create_if_missing
59+
end
60+
61+
bash "install efs utils" do
62+
cwd node['cluster']['sources_dir']
63+
code adc_install_script_code(efs_utils_tarball, package_name, package_version)
64+
end
65+
66+
else
67+
# Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
68+
bash "install efs utils" do
69+
cwd node['cluster']['sources_dir']
70+
code install_script_code(efs_utils_tarball, package_name, package_version)
71+
end
72+
end
73+
74+
action_increase_poll_interval
75+
end

‎cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,13 @@
1212
# or in the "LICENSE.txt" file accompanying this file.
1313
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
1414
# See the License for the specific language governing permissions and limitations under the License.
15+
package_name = "amazon-efs-utils"
1516

1617
action :install_utils do
18+
package_version = new_resource.efs_utils_version
19+
efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
20+
efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
21+
1722
package_repos 'update package repositories' do
1823
action :update
1924
end
@@ -29,11 +34,6 @@
2934

3035
return if redhat_on_docker?
3136

32-
package_name = "amazon-efs-utils"
33-
package_version = _efs_utils_version
34-
efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
35-
efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz"
36-
3737
# Do not install efs-utils if a same or newer version is already installed.
3838
return if already_installed?(package_name, package_version)
3939

@@ -46,14 +46,26 @@
4646
mode '0644'
4747
retries 3
4848
retry_delay 5
49-
checksum _efs_utils_checksum
49+
checksum new_resource.efs_utils_checksum
5050
action :create_if_missing
5151
end
5252

53+
action_install_efs_utils
54+
5355
# Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html
56+
# bash "install efs utils" do
57+
# cwd node['cluster']['sources_dir']
58+
# code install_script_code(efs_utils_tarball, package_name, package_version)
59+
# end
60+
action_increase_poll_interval
61+
end
62+
63+
action :install_efs_utils do
64+
package_version = new_resource.efs_utils_version
65+
efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz"
66+
5467
bash "install efs utils" do
5568
cwd node['cluster']['sources_dir']
5669
code install_script_code(efs_utils_tarball, package_name, package_version)
5770
end
58-
action_increase_poll_interval
5971
end

‎cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@
1212
# limitations under the License.
1313

1414
virtualenv_path = cookbook_virtualenv_path
15-
pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
15+
dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}"
16+
pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{dependency_package_name}.tgz"
1617
if platform?('amazon') && node['platform_version'] == "2"
18+
dependency_package_name = "dependencies"
1719
pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
1820
end
1921

@@ -31,22 +33,24 @@
3133
not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") }
3234
end
3335

34-
remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
35-
source pypi_s3_uri
36-
mode '0644'
37-
retries 3
38-
retry_delay 5
39-
action :create_if_missing
40-
end
36+
if aws_region.start_with?("us-iso")
37+
remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
38+
source pypi_s3_uri
39+
mode '0644'
40+
retries 3
41+
retry_delay 5
42+
action :create_if_missing
43+
end
4144

42-
bash 'pip install' do
43-
user 'root'
44-
group 'root'
45-
cwd "#{node['cluster']['base_dir']}"
46-
code <<-REQ
45+
bash 'pip install' do
46+
user 'root'
47+
group 'root'
48+
cwd "#{node['cluster']['base_dir']}"
49+
code <<-REQ
4750
set -e
4851
tar xzf cookbook-dependencies.tgz
49-
cd dependencies
52+
cd #{dependency_package_name}
5053
#{virtualenv_path}/bin/pip install * -f ./ --no-index
5154
REQ
55+
end
5256
end

‎cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66
cached(:python_version) { 'python_version' }
77
cached(:system_pyenv_root) { 'system_pyenv_root' }
88
cached(:virtualenv_path) { 'system_pyenv_root/versions/python_version/envs/cookbook_virtualenv' }
9+
cached(:aws_region) { 'us-iso-test' }
910

1011
context "when cookbook virtualenv not installed yet" do
1112
cached(:chef_run) do
1213
runner = runner(platform: platform, version: version) do |node|
14+
allow_any_instance_of(Object).to receive(:aws_region).and_return(aws_region)
1315
node.override['cluster']['system_pyenv_root'] = system_pyenv_root
1416
node.override['cluster']['python-version'] = python_version
17+
node.override['cluster']['region'] = aws_region
1518
end
1619
runner.converge(described_recipe)
1720
end
@@ -32,13 +35,14 @@
3235
expect(node.default['cluster']['cookbook_virtualenv_path']).to eq(virtualenv_path)
3336
is_expected.to write_node_attributes('dump node attributes')
3437
end
35-
36-
it 'installs python packages' do
37-
is_expected.to run_bash("pip install").with(
38-
user: 'root',
39-
group: 'root',
40-
cwd: "#{node['cluster']['base_dir']}"
41-
).with_code(/tar xzf cookbook-dependencies.tgz/)
38+
context "when in isolated region" do
39+
it 'installs python packages' do
40+
is_expected.to run_bash("pip install").with(
41+
user: 'root',
42+
group: 'root',
43+
cwd: "#{node['cluster']['base_dir']}"
44+
).with_code(/tar xzf cookbook-dependencies.tgz/)
45+
end
4246
end
4347
end
4448
end

‎util/upload-cookbook.sh‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -93,21 +93,22 @@ main() {
9393
# Create archive and md5
9494
_cwd=$(pwd)
9595
pushd "${_srcdir}" > /dev/null || exit
96+
GIT_REF=$(git rev-parse HEAD)
9697
_stashName=$(git stash create)
97-
git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz"
98+
git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz"
9899
#tar zcvf "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" --transform "s,^aws-parallelcluster-cookbook/,aws-parallelcluster-cookbook-${_version}/," ../aws-parallelcluster-cookbook
99100
popd > /dev/null || exit
100-
md5sum aws-parallelcluster-cookbook-${_version}.tgz > aws-parallelcluster-cookbook-${_version}.md5
101+
md5sum aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5
101102

102103
# upload packages
103104
_key_path="parallelcluster/${_version}/cookbooks"
104105
if [ -n "${_scope}" ]; then
105106
_key_path="${_key_path}/${_scope}"
106107
fi
107-
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz || _error_exit 'Failed to push cookbook to S3'
108-
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.md5 || _error_exit 'Failed to push cookbook md5 to S3'
109-
aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to fetch LastModified date'
110-
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to push cookbook date'
108+
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz || _error_exit 'Failed to push cookbook to S3'
109+
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 || _error_exit 'Failed to push cookbook md5 to S3'
110+
aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to fetch LastModified date'
111+
aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to push cookbook date'
111112

112113
_bucket_region=$(aws ${_profile} s3api get-bucket-location --bucket ${_bucket} --output text)
113114
if [ ${_bucket_region} = "None" ]; then

0 commit comments

Comments
 (0)