Skip to content

Implement GCP container deployments #275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
toooadi wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Implement GCP container deployments #275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2000ec5
Import PyPy/GCP files from features/pypy-runtime
toooadi Jan 5, 2026
36a431c
Small fixes
toooadi Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 128 additions & 19 deletions config/systems.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,41 @@
],
"packages": []
}
},
"pypy": {
"base_images": {
"x64": {
"3.11": "pypy:3.11"
},
"arm64": {
"3.11": "pypy:3.11"
}
},
"images": [
"run",
"build"
],
"username": "docker_user",
"deployment": {
"files": [
"storage.py",
"nosql.py"
],
"packages": [],
"module_packages": {
"nosql": [
"boto3==1.28.3"
]
}
}
}
},
"architecture": ["x64"],
"deployments": ["package"]
"architecture": [
"x64"
],
"deployments": [
"package"
]
},
"aws": {
"languages": {
Expand Down Expand Up @@ -121,10 +152,39 @@
"uuid": "3.4.0"
}
}
},
"pypy": {
"base_images": {
"x64": {
"3.11": "pypy:3.11-slim"
},
"arm64": {
"3.11": "pypy:3.11-slim"
}
},
"images": [
"build"
],
"deployment": {
"files": [
"handler.py",
"storage.py",
"nosql.py",
"setup.py"
],
"packages": [],
"module_packages": {}
}
}
},
"architecture": ["x64", "arm64"],
"deployments": ["package", "container"]
"architecture": [
"x64",
"arm64"
],
"deployments": [
"package",
"container"
]
},
"azure": {
"languages": {
Expand Down Expand Up @@ -188,19 +248,26 @@
"username": "docker_user"
}
},
"architecture": ["x64"],
"deployments": ["package"]
"architecture": [
"x64"
],
"deployments": [
"package"
]
},
"gcp": {
"languages": {
"python": {
"base_images": {
"x64": {
"3.8": "ubuntu:22.04",
"3.9": "ubuntu:22.04",
"3.10": "ubuntu:22.04",
"3.11": "ubuntu:22.04",
"3.12": "ubuntu:22.04"
"3.10": "python:3.10-slim",
"3.11": "python:3.11-slim",
"3.12": "python:3.12-slim"
},
"arm64": {
"3.10": "python:3.10-slim",
"3.11": "python:3.11-slim",
"3.12": "python:3.12-slim"
}
},
"images": [
Expand All @@ -211,7 +278,39 @@
"files": [
"handler.py",
"storage.py",
"nosql.py"
"nosql.py",
"setup.py"
],
"packages": [],
"module_packages": {
"storage": [
"google-cloud-storage"
],
"nosql": [
"google-cloud-datastore"
]
}
}
},
"pypy": {
"base_images": {
"x64": {
"3.11": "pypy:3.11-slim"
},
"arm64": {
"3.11": "pypy:3.11-slim"
}
},
"images": [
"build"
],
"username": "docker_user",
"deployment": {
"files": [
"handler.py",
"storage.py",
"nosql.py",
"setup.py"
],
"packages": [],
"module_packages": {
Expand All @@ -227,8 +326,8 @@
"nodejs": {
"base_images": {
"x64": {
"18": "ubuntu:22.04",
"20": "ubuntu:22.04"
"20": "node:20-slim",
"22": "node:22-slim"
}
},
"images": [
Expand All @@ -252,8 +351,14 @@
"username": "docker_user"
}
},
"architecture": ["x64"],
"deployments": ["package"]
"architecture": [
"x64",
"arm64"
],
"deployments": [
"package",
"container"
]
},
"openwhisk": {
"languages": {
Expand Down Expand Up @@ -313,7 +418,11 @@
}
}
},
"architecture": ["x64"],
"deployments": ["container"]
"architecture": [
"x64"
],
"deployments": [
"container"
]
}
}
}
6 changes: 1 addition & 5 deletions dockerfiles/gcp/nodejs/Dockerfile.build
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
ARG BASE_IMAGE
FROM ${BASE_IMAGE}
ARG VERSION
ENV NVM_DIR=/nvm

#RUN install_node --ignore-verification-failure v${VERSION}
RUN apt-get update && apt-get install -y gosu wget
RUN mkdir -p ${NVM_DIR} && wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
RUN . ${NVM_DIR}/nvm.sh && nvm install ${VERSION} && nvm alias default ${VERSION} && nvm use default
COPY --from=tianon/gosu:1.19-debian /usr/local/bin/gosu /usr/local/bin/gosu

RUN mkdir -p /sebs/
COPY dockerfiles/nodejs_installer.sh /sebs/installer.sh
Expand Down
7 changes: 3 additions & 4 deletions dockerfiles/gcp/python/Dockerfile.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@ ENV PYTHON_VERSION=${VERSION}
ENV DEBIAN_FRONTEND="noninteractive"
ENV TZ="Europe/Zurich"

COPY --from=tianon/gosu:1.19-debian /usr/local/bin/gosu /usr/local/bin/gosu

RUN apt-get update\
&& apt-get install -y --no-install-recommends gosu gcc build-essential libxml2 libxml2-dev zlib1g-dev software-properties-common gpg-agent zip\
&& add-apt-repository -y ppa:deadsnakes/ppa\
&& apt-get update\
&& apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-venv python${PYTHON_VERSION}-dev\
&& apt-get install -y --no-install-recommends gcc build-essential libxml2 libxml2-dev zlib1g-dev gpg-agent zip\
&& apt-get purge -y --auto-remove

#RUN export PATH=/opt/python3.7/bin:/opt/python3.6/bin:/opt/python3.5/bin:/opt/python3.4/bin:$PATH
Expand Down
62 changes: 61 additions & 1 deletion sebs/gcp/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import json
import os
from typing import cast, List, Optional, Tuple
import time
from googleapiclient.errors import HttpError

from sebs.cache import Cache
from sebs.faas.config import Config, Credentials, Resources
Expand Down Expand Up @@ -108,6 +110,7 @@ def update_cache(self, cache: Cache):
class GCPResources(Resources):
def __init__(self):
super().__init__(name="gcp")
self._container_repository = None

@staticmethod
def initialize(res: Resources, dct: dict):
Expand All @@ -120,7 +123,9 @@ def initialize(res: Resources, dct: dict):
"""

def serialize(self) -> dict:
return super().serialize()
out = super().serialize()
out["container_repository"] = self._container_repository
return out

@staticmethod
def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> "Resources":
Expand All @@ -147,6 +152,61 @@ def deserialize(config: dict, cache: Cache, handlers: LoggingHandlers) -> "Resou
def update_cache(self, cache: Cache):
super().update_cache(cache)

@property
def container_repository(self) -> str:
return self._container_repository

def check_container_repository_exists(self, config: Config, ar_client):
try:
parent = f"projects/{config.credentials.project_name}/locations/{config.region}"
repo_full_name = f"{parent}/repositories/{self._container_repository}"
self.logging.info("Checking if container repository exists...")
ar_client.projects().locations().repositories().get(name=repo_full_name).execute()
return True
except HttpError as e:
if e.resp.status == 404:
self.logging.error("Container repository does not exist.")
return False
else:
raise e

def create_container_repository(self, ar_client, parent):
request_body = {
"format": "DOCKER",
"description": "Container repository for SEBS"
}
self._container_repository = f"sebs-benchmarks-{self._resources_id}"
operation = ar_client.projects().locations().repositories().create(
parent=parent,
body=request_body,
repositoryId=self._container_repository
).execute()

while True:
# Operations for AR are global or location specific
op_name = operation['name']
op = ar_client.projects().locations().operations().get(name=op_name).execute()

if op.get('done'):
if 'error' in op:
raise Exception(f"Failed to create repo: {op['error']}")
self.logging.info("Repository created successfully.")
break
time.sleep(2)

def get_container_repository(self, config: Config, ar_client):
if self._container_repository is not None:
return self._container_repository

self._container_repository = f"sebs-benchmarks-{self._resources_id}"
if self.check_container_repository_exists(config, ar_client):
return self._container_repository

parent = f"projects/{config.credentials.project_name}/locations/{config.region}"
self.create_container_repository(ar_client, parent)
return self._container_repository



"""
FaaS system config defining cloud region (if necessary), credentials and
Expand Down
Loading