✨ Use 'build-all-images' Makefile goal in github action

nielsborie · Mar 2, 2024 · 4c84977 · 4c84977
1 parent f653db9
commit 4c84977
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 38 deletions.
diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml
@@ -6,21 +6,13 @@ on:
   pull_request:
 
 jobs:
-  build-and-push-docker-image:
-    name: Build Docker image and push to repositories
+  build-and-push-docker-images:
+    name: Build and push Docker images
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Prepare tag
-        id: prep
-        run: |
-          DOCKER_IMAGE=nielsborie/machine-learning-environments
-          VERSION=`cat ./SNAPSHOT.txt`
-          TAGS="${DOCKER_IMAGE}:${VERSION}"
-          echo ::set-output name=tags::${TAGS}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:
@@ -35,18 +27,15 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v5
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: "{{defaultContext}}:advanced"
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Clean up unnecessary directories
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Build and push image
+        run: |
+          make build-all-images
 
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/Makefile b/Makefile
@@ -1,28 +1,31 @@
 SHELL := /bin/bash
-BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 PROJECT_NAME ?= machine-learning-environments
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
 PYTHON_INTERPRETER = python3
 PYTHON_VERSION ?= 3.11
 OS_NAME = $(shell uname)
 REGISTRY_URL ?= nielsborie
+LAYER ?= base
+BUILDER ?= conda
+
+ifdef GITHUB_ACTIONS
+	BRANCH_NAME ?= $(shell echo "${GITHUB_REF}" | awk -F'/' '{print $$3}')
+else
+	BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2)
+endif
 
 ifeq ($(BRANCH_NAME),main)
 	IMAGE_VERSION := v$(shell cat ./VERSION.txt)
 else ifeq ($(BRANCH_NAME),develop)
 	IMAGE_VERSION := v$(shell cat ./SNAPSHOT.txt)
 else
-	IMAGE_VERSION := -$(BRANCH_NAME)
+	IMAGE_VERSION := $(BRANCH_NAME)
 endif
-DOCKER_TAG_NAME := ${LAYER}-py${PYTHON_VERSION}-${IMAGE_VERSION}
 
 SUPPORTED_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12
 ALL_LAYERS := base advanced
 ALL_BUILDERS := conda mamba
 
-LAYER ?= base
-BUILDER ?= conda
-
 .DEFAULT_GOAL:=help
 
 help:  ## Display this help
@@ -45,18 +48,18 @@ build-all-images: ## Build all machine-learning-environments docker images
 build-image: ## Build a single machine-learning-environments docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
 	@echo "PYTHON_VERSION=$(PYTHON_VERSION) PYTHON_RELEASE_VERSION=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json)"
 	@real_python_version=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json); \
-	docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(PROJECT_NAME):$${LAYER}-$(BUILDER)-py$(PYTHON_VERSION)-$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/
+	docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/
 
 
-docker-push: ## Push machine-learning-environments image to registry
-	docker push ${REGISTRY_URL}/${PROJECT_NAME}:${DOCKER_TAG_NAME}
+docker-push: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
+	docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)
 	if [ "${BRANCH_NAME}" = "main" ]; then \
-        docker push ${REGISTRY_URL}/${PROJECT_NAME}:latest; \
+        docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \
     fi;
 
 ### Running environments ###
 docker-run: ## Run machine-learning-environments using docker image  (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
-	docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(PROJECT_NAME):$${LAYER}-$(BUILDER)-py$(PYTHON_VERSION)-$(IMAGE_VERSION)
+	docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)
 
 docker-interactive: ## Enter into the machine-learning-environments container
 	docker exec -it ML-env /bin/bash

diff --git a/layers/advanced/conda.Dockerfile b/layers/advanced/conda.Dockerfile
@@ -4,7 +4,7 @@ ARG PYTHON_RELEASE_VERSION
 ARG IMAGE_VERSION
 ARG BUILDER
 
-FROM nielsborie/machine-learning-environments:base-${BUILDER}-py${PYTHON_VERSION}-${IMAGE_VERSION} as base
+FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base
 
 ARG PYTHON_VERSION
 ARG PYTHON_RELEASE_VERSION
@@ -31,7 +31,7 @@ ENV PATH="/opt/openjdk-17/bin:${PATH}"
 ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server"
 
 # Install additional Python packages and configure Java
-RUN /opt/conda/bin/conda run -n ${ENV_NAME} pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && \
+RUN /opt/conda/bin/conda run -n ${ENV_NAME} pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir && \
     rm -rf /root/.cache/pip/*
 
 # Set the entry point

diff --git a/layers/advanced/mamba.Dockerfile b/layers/advanced/mamba.Dockerfile
@@ -4,7 +4,7 @@ ARG PYTHON_RELEASE_VERSION
 ARG IMAGE_VERSION
 ARG BUILDER
 
-FROM nielsborie/machine-learning-environments:base-${BUILDER}-py${PYTHON_VERSION}-${IMAGE_VERSION} as base
+FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base
 
 ARG PYTHON_VERSION
 ARG PYTHON_RELEASE_VERSION
@@ -31,7 +31,7 @@ ENV PATH="/opt/openjdk-17/bin:${PATH}"
 ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server"
 
 # Install additional Python packages and configure Java
-RUN micromamba run -n py3.9 pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o
+RUN micromamba run -n py3.9 pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir
 
 USER root
 RUN rm -rf /root/.cache/pip/*

diff --git a/layers/base/mamba.Dockerfile b/layers/base/mamba.Dockerfile
@@ -16,4 +16,6 @@ RUN micromamba config set extract_threads 1 && \
 
 ENV PATH /opt/conda/envs/${ENV_NAME}/bin:$PATH
 
+WORKDIR /home/mambauser
+
 ENTRYPOINT ["/bin/bash"]
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,4 +16,6 @@ RUN micromamba config set extract_threads 1 && \

		ENV PATH /opt/conda/envs/${ENV_NAME}/bin:$PATH

		WORKDIR /home/mambauser

		ENTRYPOINT ["/bin/bash"]