diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 19d065c..e5533c9 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -6,21 +6,13 @@ on: pull_request: jobs: - build-and-push-docker-image: - name: Build Docker image and push to repositories + build-and-push-docker-images: + name: Build and push Docker images runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Prepare tag - id: prep - run: | - DOCKER_IMAGE=nielsborie/machine-learning-environments - VERSION=`cat ./SNAPSHOT.txt` - TAGS="${DOCKER_IMAGE}:${VERSION}" - echo ::set-output name=tags::${TAGS} - - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: @@ -35,18 +27,15 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v5 - with: - builder: ${{ steps.buildx.outputs.name }} - context: "{{defaultContext}}:advanced" - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max + - name: Clean up unnecessary directories + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Build and push image + run: | + make build-all-images - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Makefile b/Makefile index 29b9d5a..e0ad540 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,31 @@ SHELL := /bin/bash -BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2) PROJECT_NAME ?= machine-learning-environments GIT_COMMIT ?= $(shell git rev-parse HEAD) PYTHON_INTERPRETER = python3 PYTHON_VERSION ?= 3.11 OS_NAME = $(shell uname) REGISTRY_URL ?= nielsborie +LAYER ?= base +BUILDER ?= conda + +ifdef GITHUB_ACTIONS + BRANCH_NAME ?= $(shell echo "${GITHUB_REF}" | awk -F'/' '{print $$3}') +else + BRANCH_NAME ?= $(shell git branch | grep \* | cut -d ' ' -f2) +endif ifeq ($(BRANCH_NAME),main) IMAGE_VERSION := v$(shell cat ./VERSION.txt) else ifeq ($(BRANCH_NAME),develop) IMAGE_VERSION := v$(shell cat ./SNAPSHOT.txt) else - IMAGE_VERSION := -$(BRANCH_NAME) + IMAGE_VERSION := $(BRANCH_NAME) endif -DOCKER_TAG_NAME := ${LAYER}-py${PYTHON_VERSION}-${IMAGE_VERSION} SUPPORTED_PYTHON_VERSIONS := 3.9 3.10 3.11 3.12 ALL_LAYERS := base advanced ALL_BUILDERS := conda mamba -LAYER ?= base -BUILDER ?= conda - .DEFAULT_GOAL:=help help: ## Display this help @@ -45,18 +48,18 @@ build-all-images: ## Build all machine-learning-environments docker images build-image: ## Build a single machine-learning-environments docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) @echo "PYTHON_VERSION=$(PYTHON_VERSION) PYTHON_RELEASE_VERSION=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json)" @real_python_version=$$(jq -r '.python."$(PYTHON_VERSION)".release' package.json); \ - docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(PROJECT_NAME):$${LAYER}-$(BUILDER)-py$(PYTHON_VERSION)-$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ + docker build --progress=plain --no-cache --force-rm -t $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) --build-arg PYTHON_RELEASE_VERSION=$$real_python_version --build-arg PYTHON_VERSION=$(PYTHON_VERSION) --build-arg IMAGE_VERSION=$(IMAGE_VERSION) --build-arg BUILDER=$(BUILDER) -f layers/$(LAYER)/$(BUILDER).Dockerfile layers/$(LAYER)/ -docker-push: ## Push machine-learning-environments image to registry - docker push ${REGISTRY_URL}/${PROJECT_NAME}:${DOCKER_TAG_NAME} +docker-push: ## Push machine-learning-environments image to registry (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) + docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) if [ "${BRANCH_NAME}" = "main" ]; then \ - docker push ${REGISTRY_URL}/${PROJECT_NAME}:latest; \ + docker push $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):latest; \ fi; ### Running environments ### docker-run: ## Run machine-learning-environments using docker image (args : PYTHON_VERSION, LAYER, BUILDER, IMAGE_VERSION) - docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(PROJECT_NAME):$${LAYER}-$(BUILDER)-py$(PYTHON_VERSION)-$(IMAGE_VERSION) + docker run --rm -it -d --name ML-env $(REGISTRY_URL)/$(LAYER)-$(BUILDER)-py$(PYTHON_VERSION):$(IMAGE_VERSION) docker-interactive: ## Enter into the machine-learning-environments container docker exec -it ML-env /bin/bash diff --git a/layers/advanced/conda.Dockerfile b/layers/advanced/conda.Dockerfile index c8195d1..63fddbc 100644 --- a/layers/advanced/conda.Dockerfile +++ b/layers/advanced/conda.Dockerfile @@ -4,7 +4,7 @@ ARG PYTHON_RELEASE_VERSION ARG IMAGE_VERSION ARG BUILDER -FROM nielsborie/machine-learning-environments:base-${BUILDER}-py${PYTHON_VERSION}-${IMAGE_VERSION} as base +FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base ARG PYTHON_VERSION ARG PYTHON_RELEASE_VERSION @@ -31,7 +31,7 @@ ENV PATH="/opt/openjdk-17/bin:${PATH}" ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server" # Install additional Python packages and configure Java -RUN /opt/conda/bin/conda run -n ${ENV_NAME} pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o && \ +RUN /opt/conda/bin/conda run -n ${ENV_NAME} pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir && \ rm -rf /root/.cache/pip/* # Set the entry point diff --git a/layers/advanced/mamba.Dockerfile b/layers/advanced/mamba.Dockerfile index 097a73b..c6a3a32 100644 --- a/layers/advanced/mamba.Dockerfile +++ b/layers/advanced/mamba.Dockerfile @@ -4,7 +4,7 @@ ARG PYTHON_RELEASE_VERSION ARG IMAGE_VERSION ARG BUILDER -FROM nielsborie/machine-learning-environments:base-${BUILDER}-py${PYTHON_VERSION}-${IMAGE_VERSION} as base +FROM nielsborie/base-${BUILDER}-py${PYTHON_VERSION}:${IMAGE_VERSION} as base ARG PYTHON_VERSION ARG PYTHON_RELEASE_VERSION @@ -31,7 +31,7 @@ ENV PATH="/opt/openjdk-17/bin:${PATH}" ENV LD_LIBRARY_PATH="/opt/openjdk-17/lib/server" # Install additional Python packages and configure Java -RUN micromamba run -n py3.9 pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o +RUN micromamba run -n py3.9 pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --no-cache-dir USER root RUN rm -rf /root/.cache/pip/* diff --git a/layers/base/mamba.Dockerfile b/layers/base/mamba.Dockerfile index ddff6f7..bec3356 100644 --- a/layers/base/mamba.Dockerfile +++ b/layers/base/mamba.Dockerfile @@ -16,4 +16,6 @@ RUN micromamba config set extract_threads 1 && \ ENV PATH /opt/conda/envs/${ENV_NAME}/bin:$PATH +WORKDIR /home/mambauser + ENTRYPOINT ["/bin/bash"] \ No newline at end of file