diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index e5533c9..f15cb85 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -10,32 +10,31 @@ jobs: name: Build and push Docker images runs-on: ubuntu-latest steps: - - name: Checkout code + - name: ↙️ Checkout code uses: actions/checkout@v4 - - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: platforms: all - - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Clean up unnecessary directories + - name: ✨ Clean up unnecessary directories run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Build and push image + - name: ⚙️📤 Build and push image run: | - make build-all-images - - - name: Image digest + if [ "${{ github.ref }}" != 'refs/heads/main' ] && [ "${{ github.ref }}" != 'refs/heads/develop' ]; then + SUPPORTED_PYTHON_VERSIONS="3.9 3.12" + fi + make build-and-push-all-images + - name: 🎉 Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Makefile b/Makefile index e0ad540..60c0f4f 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,15 @@ SUPPORTED_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 ALL_LAYERS := base advanced ALL_BUILDERS := conda mamba +ARCHITECTURE := $(shell uname -m) + +ifeq ($(ARCHITECTURE),arm64) + DOCKER_BUILD_COMMAND = docker build --progress=plain --no-cache --force-rm +else + DOCKER_BUILD_COMMAND = docker buildx build --platform linux/amd64,linux/arm64 --progress=plain --no-cache --force-rm +endif + + .DEFAULT_GOAL:=help help: ## Display this help @@ -38,25 +47,55 @@ help: ## Display this help ### Docker ### .PHONY: build-all-images build-image +build-and-push-all-images: ## Build all machine-learning-environments docker images and push them to the registry + @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + if [ "$(BRANCH_NAME)" = "develop" ] || [ "$(BRANCH_NAME)" = "main" ]; then \ + $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + fi \ + done; \ + $(MAKE) clean-images; \ + done \ + done + +clean-images: ## Clean all built images and associated layers + @echo "Cleaning built images and associated layers..." + @docker image prune -af + build-all-images: ## Build all machine-learning-environments docker images @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ - for layer in $(ALL_LAYERS); do \ - $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer IMAGE_VERSION=$(IMAGE_VERSION); \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + done \ + done \ + done + +push-all-images: ## Push all machine-learning-environments docker images to the registry + @for version in $(SUPPORTED_PYTHON_VERSIONS); do \ + for builder in $(ALL_BUILDERS); do \ + for layer in $(ALL_LAYERS); do \ + $(MAKE) push-image PYTHON_VERSION=$$version LAYER=$$layer BUILDER=$$builder IMAGE_VERSION=$(IMAGE_VERSION); \ + done \ done \ done build-image: ## Build a single machine-learning-environments docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) - @echo "PYTHON_VERSION=$(PYTHON_VERSION) PYTHON_RELEASE_VERSION=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json)" + @echo "Building image using PYTHON_VERSION=$(PYTHON_VERSION) LAYER=$(LAYER) BUILDER=$(BUILDER) IMAGE_VERSION=$(IMAGE_VERSION)" @real_python_version=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json); \ - docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ - + $(DOCKER_BUILD_COMMAND) -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ -docker-push: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) +push-image: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) + @echo "Pushing image $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)" docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) if [ "${BRANCH_NAME}" = "main" ]; then \ docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \ fi; + + ### Running environments ### docker-run: ## Run machine-learning-environments using docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)