From fc5e4e57d0034f92180c3e818c4bfbd4e5dc7017 Mon Sep 17 00:00:00 2001 From: "Borie, Niels" Date: Mon, 4 Mar 2024 09:45:00 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8Enhance=20(docker)=20image=20push?= =?UTF-8?q?=20capabilities=20in=20the=20Makefile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e0ad540..32328a9 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,16 @@ help: ## Display this help build-all-images: ## Build all machine-learning-environments docker images @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ for layer in $(ALL_LAYERS); do \ - $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer IMAGE_VERSION=$(IMAGE_VERSION); \ + for builder in $(ALL_BUILDERS); do \ + $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + done \ + done \ + done + +push-all-images: ## push all machine-learning-environments docker images + @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer IMAGE_VERSION=$(IMAGE_VERSION); \ done \ done @@ -51,12 +60,14 @@ build-image: ## Build a single machine-learning-environments docker image (args docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ -docker-push: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) +push-image: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) if [ "${BRANCH_NAME}" = "main" ]; then \ docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \ fi; + + ### Running environments ### docker-run: ## Run machine-learning-environments using docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) From f56447f6338e2b2055044365ef36b2b421065ac1 Mon Sep 17 00:00:00 2001 From: "Borie, Niels" Date: Mon, 4 Mar 2024 09:45:52 +0100 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9C=A8Add=20'Push=20images'=20stage=20in?= =?UTF-8?q?=20the=20CI=20GitHub=20action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build_and_publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index e5533c9..60dfe15 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -12,30 +12,30 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: platforms: all - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Clean up unnecessary directories + - name: ✨Clean up unnecessary directories run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Build and push image + - name: ⚙️ Build and push image run: | make build-all-images - - - name: Image digest + - name: 📤 Push images + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' + run: | + make push-all-images + - name: 🎉 Image digest run: echo ${{ steps.docker_build.outputs.digest }} From f2abd1ceee6fb1e9f22aa8fc62cc0cec4f99f8cc Mon Sep 17 00:00:00 2001 From: "Borie, Niels" Date: Tue, 5 Mar 2024 20:18:31 +0100 Subject: [PATCH 3/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20=20Add=20a=20Makefile?= =?UTF-8?q?=20goal=20to=20build=20and=20push=20docker=20image=20at=20the?= =?UTF-8?q?=20same=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build_and_publish.yml | 4 +-- Makefile | 44 ++++++++++++++++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 60dfe15..50ac940 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -10,7 +10,7 @@ jobs: name: Build and push Docker images runs-on: ubuntu-latest steps: - - name: Checkout code + - name: ↙️ Checkout code uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -24,7 +24,7 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: ✨Clean up unnecessary directories + - name: ✨ Clean up unnecessary directories run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc diff --git a/Makefile b/Makefile index 32328a9..60c0f4f 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,15 @@ SUPPORTED_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 ALL_LAYERS := base advanced ALL_BUILDERS := conda mamba +ARCHITECTURE := $(shell uname -m) + +ifeq ($(ARCHITECTURE),arm64) + DOCKER_BUILD_COMMAND = docker build --progress=plain --no-cache --force-rm +else + DOCKER_BUILD_COMMAND = docker buildx build --platform linux/amd64,linux/arm64 --progress=plain --no-cache --force-rm +endif + + .DEFAULT_GOAL:=help help: ## Display this help @@ -38,29 +47,48 @@ help: ## Display this help ### Docker ### .PHONY: build-all-images build-image +build-and-push-all-images: ## Build all machine-learning-environments docker images and push them to the registry + @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + if [ "$(BRANCH_NAME)" = "develop" ] || [ "$(BRANCH_NAME)" = "main" ]; then \ + $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + fi \ + done; \ + $(MAKE) clean-images; \ + done \ + done + +clean-images: ## Clean all built images and associated layers + @echo "Cleaning built images and associated layers..." + @docker image prune -af + build-all-images: ## Build all machine-learning-environments docker images @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ - for layer in $(ALL_LAYERS); do \ - for builder in $(ALL_BUILDERS); do \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ done \ done \ done -push-all-images: ## push all machine-learning-environments docker images +push-all-images: ## Push all machine-learning-environments docker images to the registry @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ - for layer in $(ALL_LAYERS); do \ - $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer IMAGE_VERSION=$(IMAGE_VERSION); \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + done \ done \ done build-image: ## Build a single machine-learning-environments docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) - @echo "PYTHON_VERSION=$(PYTHON_VERSION) PYTHON_RELEASE_VERSION=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json)" + @echo "Building image using PYTHON_VERSION=$(PYTHON_VERSION) LAYER=$(LAYER) BUILDER=$(BUILDER) IMAGE_VERSION=$(IMAGE_VERSION)" @real_python_version=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json); \ - docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ - + $(DOCKER_BUILD_COMMAND) -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ push-image: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) + @echo "Pushing image $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)" docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) if [ "${BRANCH_NAME}" = "main" ]; then \ docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \ From cb46ddff1c4f3613b7973efba0d4a7f8e32a0aae Mon Sep 17 00:00:00 2001 From: "Borie, Niels" Date: Tue, 5 Mar 2024 21:54:43 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=92=9A=20Build=203.9=20and=203.12=20d?= =?UTF-8?q?ocker=20images=20on=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build_and_publish.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 50ac940..f15cb85 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -30,12 +30,11 @@ jobs: sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: ⚙️ Build and push image + - name: ⚙️📤 Build and push image run: | - make build-all-images - - name: 📤 Push images - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' - run: | - make push-all-images + if [ "${{ github.ref }}" != 'refs/heads/main' ] && [ "${{ github.ref }}" != 'refs/heads/develop' ]; then + SUPPORTED_PYTHON_VERSIONS="3.9 3.12" + fi + make build-and-push-all-images - name: 🎉 Image digest run: echo ${{ steps.docker_build.outputs.digest }}