From 2293f1dbdc2333e3a4f3df54ebaf0c51b00f0d91 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 08:46:03 -0700 Subject: [PATCH 01/11] apt-get clean clearing cache may have been breaking ability to find softwareproperties module --- .github/docker-images/ubuntu-18-x64/Dockerfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index c8eb699f9..bb1bcf63d 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -30,8 +30,7 @@ RUN apt-get update \ libxml2-dev \ libxmlsec1-dev \ libffi-dev \ - liblzma-dev \ - && apt-get clean + liblzma-dev # Build Python 3.10 from source RUN cd /tmp \ @@ -49,10 +48,7 @@ RUN cd /tmp \ # Install Go from default Ubuntu repository RUN apt-get update \ && apt-get -y install \ - software-properties-common \ - golang-go \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + golang-go ############################################################################### # Python/AWS CLI From ea49993aa306efa2e9a1a156a04c3267ec2f8cf9 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:04:22 -0700 Subject: [PATCH 02/11] point link to lines in a branch so they don't move around --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f8c42d41a..809882879 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,7 @@ debugging experience: * --build-dir=/path/to/other/git/repo - will jump to this directory before starting execution, helpful for debugging another project's configuration and build scripts -When debugging builder from downstream CI (AWS common runtime repos), to use a branch of builder, you will need to change `BUILDER_VERSION` to the branch name of aws-crt-builder, and the `BUILDER_SOURCE` to be `channels` from `ci.yml`. Eg: for `aws-crt-java`, make those change to the [here](https://github.com/awslabs/aws-crt-java/blob/main/.github/workflows/ci.yml#L10-L11). +When debugging builder from downstream CI (AWS common runtime repos), to use a branch of builder, you will need to change `BUILDER_VERSION` to the branch name of aws-crt-builder, and the `BUILDER_SOURCE` to be `channels` from `ci.yml`. Eg: for `aws-crt-java`, make those change to the [here](https://github.com/awslabs/aws-crt-java/blob/eb3813e4da89709fe71611fd4fdf59a5eef40336/.github/workflows/ci.yml#L10-L11). ### Docker Images Each docker image has a script which will fetch the builder app baked into it, and will then call the builder with the arguments provided. From d8c116fe242d4189158867f9e82395744db41ecf Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:04:51 -0700 Subject: [PATCH 03/11] grammer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 809882879..aece5796c 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,7 @@ debugging experience: * --build-dir=/path/to/other/git/repo - will jump to this directory before starting execution, helpful for debugging another project's configuration and build scripts -When debugging builder from downstream CI (AWS common runtime repos), to use a branch of builder, you will need to change `BUILDER_VERSION` to the branch name of aws-crt-builder, and the `BUILDER_SOURCE` to be `channels` from `ci.yml`. Eg: for `aws-crt-java`, make those change to the [here](https://github.com/awslabs/aws-crt-java/blob/eb3813e4da89709fe71611fd4fdf59a5eef40336/.github/workflows/ci.yml#L10-L11). +When debugging builder from downstream CI (AWS common runtime repos), to use a branch of builder, you will need to change `BUILDER_VERSION` to the branch name of aws-crt-builder, and the `BUILDER_SOURCE` to be `channels` from `ci.yml`. Eg: for `aws-crt-java`, make those changes [here](https://github.com/awslabs/aws-crt-java/blob/eb3813e4da89709fe71611fd4fdf59a5eef40336/.github/workflows/ci.yml#L10-L11). ### Docker Images Each docker image has a script which will fetch the builder app baked into it, and will then call the builder with the arguments provided. From 7f87aca62af9925b31b5d437427d3908ebcf957b Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:23:15 -0700 Subject: [PATCH 04/11] point to python instead of python3 and insure python3 software properties is installed --- .github/docker-images/ubuntu-18-x64/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index bb1bcf63d..dec7c00c6 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -42,12 +42,13 @@ RUN cd /tmp \ && make altinstall \ && cd / \ && rm -rf /tmp/Python-3.10.12* \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python3 \ + && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ && /usr/local/bin/python3.10 --version -# Install Go from default Ubuntu repository +# Install Go and ensure system Python packages are available RUN apt-get update \ && apt-get -y install \ + python3-software-properties \ golang-go ############################################################################### From 4aff4755d090b977e07c2928b42c45a997d82547 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:40:31 -0700 Subject: [PATCH 05/11] use sed to explicitly set to python3.6 --- .github/docker-images/ubuntu-18-x64/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index dec7c00c6..cc6b4e74f 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -45,11 +45,12 @@ RUN cd /tmp \ && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ && /usr/local/bin/python3.10 --version -# Install Go and ensure system Python packages are available +# Install Go and fix apt-add-repository for system Python RUN apt-get update \ && apt-get -y install \ python3-software-properties \ - golang-go + golang-go \ + && sed -i '1s|.*|#!/usr/bin/python3.6|' /usr/bin/apt-add-repository ############################################################################### # Python/AWS CLI From 1e5192b9c827c42e99b049d40e9759014a96f80e Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:42:34 -0700 Subject: [PATCH 06/11] python 3.10 --- .github/docker-images/ubuntu-18-x64/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index cc6b4e74f..8d1976a2a 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -45,12 +45,12 @@ RUN cd /tmp \ && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ && /usr/local/bin/python3.10 --version -# Install Go and fix apt-add-repository for system Python +# Install Go and setup Python 3.10 for system tools RUN apt-get update \ && apt-get -y install \ - python3-software-properties \ golang-go \ - && sed -i '1s|.*|#!/usr/bin/python3.6|' /usr/bin/apt-add-repository + && /usr/local/bin/python3.10 -m pip install python-apt \ + && sed -i '1s|.*|#!/usr/local/bin/python3.10|' /usr/bin/apt-add-repository ############################################################################### # Python/AWS CLI From 1c22b1d6521470505b5089eee19c0c91bd1bebc6 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:46:03 -0700 Subject: [PATCH 07/11] python 3.10 for application, python 3.6 for system tools --- .github/docker-images/ubuntu-18-x64/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index 8d1976a2a..b9ea2be51 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -45,12 +45,11 @@ RUN cd /tmp \ && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ && /usr/local/bin/python3.10 --version -# Install Go and setup Python 3.10 for system tools +# Install Go and ensure system Python tools work RUN apt-get update \ && apt-get -y install \ - golang-go \ - && /usr/local/bin/python3.10 -m pip install python-apt \ - && sed -i '1s|.*|#!/usr/local/bin/python3.10|' /usr/bin/apt-add-repository + python3-software-properties \ + golang-go ############################################################################### # Python/AWS CLI From 7ff78ec4f927aea36745af69fa8ca8535434236f Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 09:55:04 -0700 Subject: [PATCH 08/11] keep system symlink intact --- .github/docker-images/ubuntu-18-x64/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index b9ea2be51..118c929dc 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -42,7 +42,7 @@ RUN cd /tmp \ && make altinstall \ && cd / \ && rm -rf /tmp/Python-3.10.12* \ - && ln -sf /usr/local/bin/python3.10 /usr/bin/python \ + && ln -sf /usr/local/bin/python3.10 /usr/local/bin/python \ && /usr/local/bin/python3.10 --version # Install Go and ensure system Python tools work From 23c63b87594ed7cda32c7dd067e865bfa8fe71de Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 10:09:16 -0700 Subject: [PATCH 09/11] add logging to track what's happening --- .github/docker-images/ubuntu-18-x64/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index 118c929dc..1b5ee6ee1 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -49,7 +49,10 @@ RUN cd /tmp \ RUN apt-get update \ && apt-get -y install \ python3-software-properties \ - golang-go + golang-go \ + && head -1 /usr/bin/apt-add-repository \ + && python3 -c "import softwareproperties; print('softwareproperties found')" \ + && ls -la /usr/bin/python* ############################################################################### # Python/AWS CLI From 731e3f793d05ddbad7f0676f2b080d557dce6009 Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 10:15:48 -0700 Subject: [PATCH 10/11] remove log and add python3-apt --- .github/docker-images/ubuntu-18-x64/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index 1b5ee6ee1..f781e95c4 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -49,10 +49,8 @@ RUN cd /tmp \ RUN apt-get update \ && apt-get -y install \ python3-software-properties \ - golang-go \ - && head -1 /usr/bin/apt-add-repository \ - && python3 -c "import softwareproperties; print('softwareproperties found')" \ - && ls -la /usr/bin/python* + python3-apt \ + golang-go ############################################################################### # Python/AWS CLI From 169834550be3ca6eab5e759baebd513f890bf5cc Mon Sep 17 00:00:00 2001 From: Steve Kim Date: Mon, 15 Sep 2025 10:47:22 -0700 Subject: [PATCH 11/11] add software-properties-common again --- .github/docker-images/ubuntu-18-x64/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index f781e95c4..aecda3e68 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -30,6 +30,7 @@ RUN apt-get update \ libxml2-dev \ libxmlsec1-dev \ libffi-dev \ + software-properties-common \ liblzma-dev # Build Python 3.10 from source