💫 Redesign of image construction

.github/workflows/build_and_publish.yml

@@ -6,21 +6,13 @@ on:
   pull_request:
 
 jobs:
-  build-and-push-docker-image:
-    name: Build Docker image and push to repositories
+  build-and-push-docker-images:
+    name: Build and push Docker images
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Prepare tag
-        id: prep
-        run: |
-          DOCKER_IMAGE=nielsborie/machine-learning-environments
-          VERSION=`cat ./SNAPSHOT.txt`
-          TAGS="${DOCKER_IMAGE}:${VERSION}"
-          echo ::set-output name=tags::${TAGS}
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:
@@ -35,18 +27,15 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v5
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: "{{defaultContext}}:advanced"
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.prep.outputs.tags }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Clean up unnecessary directories
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - name: Build and push image
+        run: |
+          make build-all-images
 
       - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}
+        run: echo ${{ steps.docker_build.outputs.digest }}

Makefile

@@ -1,47 +1,93 @@
 SHELL := /bin/bash
-BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 PROJECT_NAME ?= machine-learning-environments
 GIT_COMMIT ?= $(shell git rev-parse HEAD)
-
+PYTHON_INTERPRETER = python3
+PYTHON_VERSION ?= 3.11
+OS_NAME = $(shell uname)
 REGISTRY_URL ?= nielsborie
-DOCKER_TAG_NAME = $(BRANCH_NAME)
-export REGISTRY_URL
+LAYER ?= base
+BUILDER ?= conda
+
+ifdef GITHUB_ACTIONS
+	BRANCH_NAME ?= $(shell echo "${GITHUB_REF}" | awk -F'/' '{print $$3}')
+else
+	BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2)
+endif
+
+ifeq ($(BRANCH_NAME),main)
+	IMAGE_VERSION := v$(shell cat ./VERSION.txt)
+else ifeq ($(BRANCH_NAME),develop)
+	IMAGE_VERSION := v$(shell cat ./SNAPSHOT.txt)
+else
+	IMAGE_VERSION := $(BRANCH_NAME)
+endif
+
+SUPPORTED_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12
+ALL_LAYERS := base advanced
+ALL_BUILDERS := conda mamba
 
 .DEFAULT_GOAL:=help
 
 help:  ## Display this help
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-10s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+	@echo "Usage:"
+	@echo "  make <target>"
+	@echo ""
+	@echo "Targets:"
+	@awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
 ### Docker ###
-docker-build: ## Build machine-learning-environments docker image
-	docker build --force-rm -t ${REGISTRY_URL}/${PROJECT_NAME}:${DOCKER_TAG_NAME} .
-	docker tag ${REGISTRY_URL}/${PROJECT_NAME}:${DOCKER_TAG_NAME} ${REGISTRY_URL}/${PROJECT_NAME}:latest
+.PHONY: build-all-images build-image
+
+build-all-images: ## Build all machine-learning-environments docker images
+	@for version in $(SUPPORTED_PYTHON_VERSIONS); do \
+		for layer in $(ALL_LAYERS); do \
+			$(MAKE) build-image PYTHON_VERSION=$$version LAYER=$$layer IMAGE_VERSION=$(IMAGE_VERSION); \
+		done \
+	done
+
+build-image: ## Build a single machine-learning-environments docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
+	@echo "PYTHON_VERSION=$(PYTHON_VERSION) PYTHON_RELEASE_VERSION=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json)"
+	@real_python_version=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json); \
+	docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/
 
-docker-push: ## Push machine-learning-environments image to registry
-	docker push ${REGISTRY_URL}/${PROJECT_NAME}:${DOCKER_TAG_NAME}
+
+docker-push: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
+	docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)
 	if [ "${BRANCH_NAME}" = "main" ]; then \
-        docker push ${REGISTRY_URL}/${PROJECT_NAME}:latest; \
+        docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \
     fi;
 
 ### Running environments ###
-docker-run: ## Run machine-learning-environments using docker image (args: version=[version])
-    ifeq ($(version),)
-    VERSION=latest
-    else
-    VERSION=$(version)
-    endif
-    #@echo "▶️ Running ${VERSION} ..."
-    docker run --name ML-env -d -p 8887:8888 ${REGISTRY_URL}/${PROJECT_NAME}:${VERSION}
+docker-run: ## Run machine-learning-environments using docker image  (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION)
+	docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION)
+
+docker-interactive: ## Enter into the machine-learning-environments container
+	docker exec -it ML-env /bin/bash
 
 start: ## Start the machine-learning-environments container
 	docker start ML-env
 
 stop: ## Stop the machine-learning-environments container
-	docker start ML-env
+	docker stop ML-env
+
+clean: ## Remove the machine-learning-environments container
+	docker rm ML-env
+
+docker-system-prune:
+	docker system prune
+
+run-within-container:  ## Execute a specified Python file within a pre-started container.  (args : SCRIPT_FILE)
+	@echo "Executing the specified Python file within a pre-started container..."
+	docker cp ${PWD}/scripts ML-env:/home
+	docker exec -it ML-env python /home/scripts/$(SCRIPT_FILE)
 
+run-in-container: ## Execute a specified Python file within a container without requiring prior startup. (args : SCRIPT_FILE)
+	@echo "Executing the specified Python file within a container without requiring prior startup..."
+	docker run -it --rm -v "${PWD}"/scripts:/home/scripts -w /home nielsborie/machine-learning-environments:${LAYER}-conda-py3.11--upgrade_and_refactos -c "python /home/scripts/$(SCRIPT_FILE)"
 
 ### RELEASE ###
-generate-changelog: ## Generate/Update CHANGELOG.md file
+## Generate/Update CHANGELOG.md file
+generate-changelog:
 	gitmoji-changelog
 
 ### GitHub action test ###

SNAPSHOT.txt

@@ -1 +1 @@
-1.0.1-SNAPSHOT
+1.0.1-snapshot

layers/advanced/conda.Dockerfile

@@ -0,0 +1,38 @@
+# Define build arguments
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE_VERSION
+ARG IMAGE_VERSION
+ARG BUILDER
+
+FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base
+
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE_VERSION
+ENV ENV_NAME=py${PYTHON_VERSION}
+
+COPY environment.yml .
+RUN sed -i "s/python=[0-9]\+\.[0-9]\+\.[0-9]\+/python=${PYTHON_RELEASE_VERSION}/" environment.yml
+RUN /opt/conda/bin/conda env update --name ${ENV_NAME} --file environment.yml --prune
+
+# Stage 2: Java installation
+FROM openjdk:17-jdk-slim as java_backend
+
+# Set Java in PATH
+ENV PATH="/usr/local/openjdk-17/bin:${PATH}"
+
+# Stage 3: Final stage
+FROM base
+
+# Copy Java binaries from java_backend stage
+COPY --from=java_backend /usr/local/openjdk-17 /opt/openjdk-17
+
+# Set Java in PATH and LD_LIBRARY_PATH
+ENV PATH="/opt/openjdk-17/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server"
+
+# Install additional Python packages and configure Java
+RUN /opt/conda/bin/conda run -n ${ENV_NAME} pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir && \
+    rm -rf /root/.cache/pip/*
+
+# Set the entry point
+ENTRYPOINT ["/bin/bash"]

layers/advanced/environment.yml

@@ -0,0 +1,37 @@
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  # version
+  - python=9.9.9
+  # vanilla
+  - xgboost
+  - lightgbm
+  - catboost
+  - mlxtend
+  - scikit-mdr
+  - vowpalwabbit
+  # interpretability
+  - shap
+  - lime
+  # nlp
+  - gensim
+  - spacy
+  # automl
+  - hyperopt
+  - optuna
+  - tpot
+  # genetic
+  - deap
+  - skrebate
+  - yellowbrick
+  - gplearn
+  # manipulation (dates, GPS, ...)
+  - gpxpy
+  - arrow
+  - haversine
+  - toolz
+  - cytoolz
+  - sacred
+  - missingno

layers/advanced/mamba.Dockerfile

@@ -0,0 +1,38 @@
+# Define build arguments
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE_VERSION
+ARG IMAGE_VERSION
+ARG BUILDER
+
+FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base
+
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE_VERSION
+ENV ENV_NAME=py${PYTHON_VERSION}
+
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml .
+RUN sed -i "s/python=[0-9]\+\.[0-9]\+\.[0-9]\+/python=${PYTHON_RELEASE_VERSION}/" environment.yml
+RUN micromamba install -y -n ${ENV_NAME} -f environment.yml
+
+# Stage 2: Java installation
+FROM openjdk:17-jdk-slim as java_backend
+
+# Set Java in PATH
+ENV PATH="/usr/local/openjdk-17/bin:${PATH}"
+
+# Stage 3: Final stage
+FROM base
+
+# Copy Java binaries from java_backend stage
+COPY --from=java_backend /usr/local/openjdk-17 /opt/openjdk-17
+
+# Set Java in PATH and LD_LIBRARY_PATH
+ENV PATH="/opt/openjdk-17/bin:${PATH}"
+ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server"
+
+# Install additional Python packages and configure Java
+RUN micromamba run -n py3.9 pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir
+
+USER root
+RUN rm -rf /root/.cache/pip/*
+USER $MAMBA_USER

layers/base/conda.Dockerfile

@@ -0,0 +1,18 @@
+FROM continuumio/miniconda3
+
+ARG PYTHON_VERSION
+ARG PYTHON_RELEASE_VERSION
+
+ENV ENV_NAME=py${PYTHON_VERSION}
+
+COPY environment.yml /opt/environment.yml
+
+RUN /opt/conda/bin/conda create -n py${PYTHON_VERSION} python=${PYTHON_RELEASE_VERSION} -y && \
+    /opt/conda/bin/conda init bash && \
+    /opt/conda/bin/conda env update -n py${PYTHON_VERSION} -f /opt/environment.yml && \
+    echo "conda activate py${PYTHON_VERSION}" >> ~/.bashrc && \
+    /opt/conda/bin/conda clean --all --yes
+
+ENV PATH=/opt/conda/envs/py${PYTHON_VERSION}/bin:$PATH
+
+ENTRYPOINT ["/bin/bash"]