Addressing package regression in Pytorch-training-2.7 image (#5478)

* Building the image and basic testing

* Addressed package regression caused by smclarify dependency s3fs

* building only training image

* explicitly declared typer, langcodes, languade_data to fix package regression

* Removed AML2_CPU_ARM64_US_EAST_1 since ami not available

* removed version pins for awscli and boto3

* added boto3 package

* removed boto3 declaration for cpu image

* added boto3 without any specific pin

* pinned awscli and boto3 to the versions in prod image

* added typer langcodes and language_data in samageker image recipe aswell

* reduced the version of awscli and boto3

* revereted awscli and boto3 to 1.42.61

* updated awscli and bot3 versions in gpu to 1.42.61 to match those in cpu

* reveted back toml file

---------

Co-authored-by: Bhanu Teja Goshikonda <[email protected]>

Author: bhanutejagk

pytorch/training/docker/2.7/py3/Dockerfile.cpu

@@ -241,11 +241,14 @@ RUN pip install --no-cache-dir -U torch==${PYTORCH_VERSION} \
     fastai==2.8.2 \
     accelerate \
     # pin numpy requirement for fastai dependency
-    # requires explicit declaration of spacy, thic, blis
+    # requires explicit declaration of spacy, thic, blis, typer, langcodes, language_data
     spacy \
     #thinc 8.3.6 is not compatible with numpy 1.26.4 (sagemaker doesn't support latest numpy)
     thinc==8.3.4  \
     blis \
+    typer \ 
+    langcodes \
+    language_data \
     numpy \
  && pip uninstall -y dataclasses
 
@@ -312,15 +315,21 @@ RUN pip install --no-cache-dir -U torch==${PYTORCH_VERSION} \
     fastai==2.8.2 \
     accelerate \
     # pin numpy requirement for fastai dependency
-    # requires explicit declaration of spacy, thic, blis
+    # requires explicit declaration of spacy, thic, blis, typer, langcodes, language_data
     spacy \
     thinc==8.3.4  \
     blis \
     numpy \
+    typer \ 
+    langcodes \
+    language_data \
  && pip uninstall -y dataclasses
 
 # Install SM packages
 RUN pip install --no-cache-dir -U \
+    # address package regression caused by smclarify depedency s3fs"
+    "awscli<=1.42.61" \
+    "boto3<=1.40.61" \
     smclarify \
     "sagemaker>=2.9.0,<3" \
     "sagemaker-experiments<1" \

pytorch/training/docker/2.7/py3/cu128/Dockerfile.gpu

@@ -143,11 +143,14 @@ RUN pip install --no-cache-dir \
     "tornado>=6.5.1" \
     "fastai==2.8.2" \
     # pin numpy requirement for fastai dependency
-    # requires explicit declaration of spacy, thic, blis
+    # requires explicit declaration of spacy, thic, blis, typer, langcodes, language_data
     spacy \
     #thinc 8.3.6 is not compatible with numpy 1.26.4 (sagemaker doesn't support latest numpy)
     "thinc==8.3.4" \
     blis \
+    typer \ 
+    langcodes \
+    language_data \
     "jinja2>=3.1.6"\
     "typing-extensions>=4.14.1" \
  && pip uninstall -y dataclasses
@@ -195,6 +198,9 @@ RUN chmod +x /usr/local/bin/start_with_right_hostname.sh
 
 # Install SM packages
 RUN pip install --no-cache-dir -U \
+    # address package regression caused by smclarify depedency s3fs"
+    "awscli<=1.42.61" \
+    "boto3<=1.40.61" \
     smclarify \
     "sagemaker>=2.9.0,<3" \
     "sagemaker-experiments<1" \

test/test_utils/__init__.py

@@ -140,10 +140,12 @@ def get_ami_id_ssm(region_name, parameter_path):
     ami_name_pattern="Deep Learning ARM64 AMI OSS Nvidia Driver GPU PyTorch 2.2.? (Ubuntu 20.04) ????????",
     IncludeDeprecated=True,
 )
+
 AML2_CPU_ARM64_US_EAST_1 = get_ami_id_boto3(
     region_name="us-east-1",
     ami_name_pattern="Deep Learning ARM64 Base OSS Nvidia Driver GPU AMI (Amazon Linux 2) ????????",
 )
+
 PT_GPU_PY3_BENCHMARK_IMAGENET_AMI_US_EAST_1 = "ami-0673bb31cc62485dd"
 PT_GPU_PY3_BENCHMARK_IMAGENET_AMI_US_WEST_2 = "ami-02d9a47bc61a31d43"