From 1b0e37c7548fc48b823b2deaafef4d6ce412b4e8 Mon Sep 17 00:00:00 2001 From: Louis Maddox Date: Tue, 18 Apr 2023 11:56:57 +0100 Subject: [PATCH] Add Python 3.10 runtime --- README.md | 19 ++++++++ dump-fs/dump-python310/handler.py | 50 +++++++++++++++++++++ dump-fs/serverless.yml | 21 +++++++++ examples/test-all.sh | 2 + runtimes/python3.10-arm64/build/Dockerfile | 25 +++++++++++ runtimes/python3.10-arm64/build/build.sh | 5 +++ runtimes/python3.10-arm64/build/publish.sh | 11 +++++ runtimes/python3.10-arm64/run/Dockerfile | 21 +++++++++ runtimes/python3.10-arm64/run/build.sh | 5 +++ runtimes/python3.10-arm64/run/publish.sh | 11 +++++ runtimes/python3.10-x86_64/build/Dockerfile | 25 +++++++++++ runtimes/python3.10-x86_64/build/build.sh | 5 +++ runtimes/python3.10-x86_64/build/publish.sh | 11 +++++ runtimes/python3.10-x86_64/run/Dockerfile | 21 +++++++++ runtimes/python3.10-x86_64/run/build.sh | 5 +++ runtimes/python3.10-x86_64/run/publish.sh | 11 +++++ 16 files changed, 248 insertions(+) create mode 100644 dump-fs/dump-python310/handler.py create mode 100644 runtimes/python3.10-arm64/build/Dockerfile create mode 100755 runtimes/python3.10-arm64/build/build.sh create mode 100755 runtimes/python3.10-arm64/build/publish.sh create mode 100644 runtimes/python3.10-arm64/run/Dockerfile create mode 100755 runtimes/python3.10-arm64/run/build.sh create mode 100755 runtimes/python3.10-arm64/run/publish.sh create mode 100644 runtimes/python3.10-x86_64/build/Dockerfile create mode 100755 runtimes/python3.10-x86_64/build/build.sh create mode 100755 runtimes/python3.10-x86_64/build/publish.sh create mode 100644 runtimes/python3.10-x86_64/run/Dockerfile create mode 100755 runtimes/python3.10-x86_64/run/build.sh create mode 100755 runtimes/python3.10-x86_64/run/publish.sh diff --git a/README.md b/README.md index 83b20f1..f4027d8 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ the [AWS CLI](https://aws.amazon.com/cli/). - [nodejs18.x](#nodejs18x) - [python3.8](#python38) - [python3.9](#python39) + - [python3.10](#python310) - [ruby2.7](#ruby27) - [java8.al2](#java8al2) - [java11](#java11) @@ -166,6 +167,24 @@ Build images | x86_64 | `mlupin/docker-lambda:python3.9-build-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.9-build-x86_64` | | arm64 | `mlupin/docker-lambda:python3.9-build-arm64` | `ghcr.io/mlupine/docker-lambda:python3.9-build-arm64` | +### python3.10 + +Runtime images + +| Platform | Docker Hub | GitHub Container Registry | +| --------- | ---------------------------------------- | ------------------------------------------------- | +| Universal | `mlupin/docker-lambda:python3.10` | `ghcr.io/mlupine/docker-lambda:python3.10` | +| x86_64 | `mlupin/docker-lambda:python3.10-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.10-x86_64` | +| arm64 | `mlupin/docker-lambda:python3.10-arm64` | `ghcr.io/mlupine/docker-lambda:python3.10-arm64` | + +Build images + +| Platform | Docker Hub | GitHub Container Registry | +| --------- | ---------------------------------------------- | ------------------------------------------------------- | +| Universal | `mlupin/docker-lambda:python3.10-build` | `ghcr.io/mlupine/docker-lambda:python3.10-build` | +| x86_64 | `mlupin/docker-lambda:python3.10-build-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.10-build-x86_64` | +| arm64 | `mlupin/docker-lambda:python3.10-build-arm64` | `ghcr.io/mlupine/docker-lambda:python3.10-build-arm64` | + ### ruby2.7 Runtime images diff --git a/dump-fs/dump-python310/handler.py b/dump-fs/dump-python310/handler.py new file mode 100644 index 0000000..63b89ad --- /dev/null +++ b/dump-fs/dump-python310/handler.py @@ -0,0 +1,50 @@ +from __future__ import print_function + +import os +import sys +import subprocess +import json +import boto3 +from boto3.s3.transfer import S3Transfer + +TRANSFER = S3Transfer(boto3.client('s3')) + +def lambda_handler_arm64(event, context): + return lambda_handler(event, context, "arm64") + +def lambda_handler_x86_64(event, context): + return lambda_handler(event, context, "x86_64") + +def lambda_handler(event, context, arch): + if 'cmd' in event: + return print(subprocess.check_output(['sh', '-c', event['cmd']]).decode('utf-8')) + + filename = f'python3.10-{arch}.tgz' + + subprocess.call(['touch', f'/tmp/{filename}']) + + subprocess.call(['sh', '-c', f'tar -cpzf /tmp/{filename} --numeric-owner --ignore-failed-read ' + + '/var/runtime /var/lang /var/rapid']) + + print('Zipping done! Uploading...') + + TRANSFER.upload_file(f'/tmp/{filename}', 'docker-lambda', + f'fs/{filename}', extra_args={'ACL': 'public-read'}) + + print('Uploading done!') + + info = {'sys.executable': sys.executable, + 'sys.argv': sys.argv, + 'sys.path': sys.path, + 'os.getcwd': os.getcwd(), + '__file__': __file__, + 'os.environ': {k: str(v) for k, v in os.environ.items()}, + 'context': {k: str(v) for k, v in context.__dict__.items()}, + 'proc environ': subprocess.check_output( + ['sh', '-c', 'echo /proc/1/environ; xargs -n 1 -0 < /proc/1/environ']).decode('utf-8').splitlines(), + 'ps aux': subprocess.check_output( + ['bash', '-O', 'extglob', '-c', 'for cmd in /proc/+([0-9])/cmdline; do echo $cmd; xargs -n 1 -0 < $cmd; done']).decode('utf-8').splitlines()} + + print(json.dumps(info, indent=2)) + + return info diff --git a/dump-fs/serverless.yml b/dump-fs/serverless.yml index bc70e6e..385bfe7 100644 --- a/dump-fs/serverless.yml +++ b/dump-fs/serverless.yml @@ -94,6 +94,27 @@ functions: layers: - { Ref: TarARMLambdaLayer } + dump-python310-x86_64: + handler: dump-python310/handler.lambda_handler_x86_64 + package: + patterns: + - "!**" + - "dump-python310/**" + runtime: python3.10 + architecture: x86_64 + layers: + - { Ref: TarX86LambdaLayer } + dump-python310-arm64: + handler: dump-python310/handler.lambda_handler_arm64 + package: + patterns: + - "!**" + - "dump-python310/**" + runtime: python3.10 + architecture: arm64 + layers: + - { Ref: TarARMLambdaLayer } + dump-ruby27-x86_64: handler: dump-ruby27/handler.lambda_handler_x86_64 package: diff --git a/examples/test-all.sh b/examples/test-all.sh index a14d63c..24614c5 100755 --- a/examples/test-all.sh +++ b/examples/test-all.sh @@ -18,10 +18,12 @@ npm test cd ${EXAMPLES_DIR}/python docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.8 lambda_function.lambda_handler docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.9 lambda_function.lambda_handler +docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.10 lambda_function.lambda_handler cd ${EXAMPLES_DIR}/python docker run --rm -it mlupin/docker-lambda:python3.8-build pip install marisa-trie docker run --rm -it mlupin/docker-lambda:python3.9-build pip install marisa-trie +docker run --rm -it mlupin/docker-lambda:python3.10-build pip install marisa-trie cd ${EXAMPLES_DIR}/ruby docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:ruby2.7 lambda_function.lambda_handler diff --git a/runtimes/python3.10-arm64/build/Dockerfile b/runtimes/python3.10-arm64/build/Dockerfile new file mode 100644 index 0000000..01666d0 --- /dev/null +++ b/runtimes/python3.10-arm64/build/Dockerfile @@ -0,0 +1,25 @@ +FROM mlupin/docker-lambda:python3.10-arm64 + +FROM mlupin/docker-lambda:build-arm64 + +ENV PATH=/var/lang/bin:$PATH \ + LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \ + AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \ + AWS_EXECUTION_ARCH=arm64 \ + PKG_CONFIG_PATH=/var/lang/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/share/pkgconfig \ + PIPX_BIN_DIR=/var/lang/bin \ + PIPX_HOME=/var/lang/pipx + +COPY --from=0 /var/runtime /var/runtime +COPY --from=0 /var/lang /var/lang +COPY --from=0 /var/rapid /var/rapid + +# Add these as a separate layer as they get updated frequently +RUN pip install -U pip setuptools wheel --no-cache-dir && \ + pip install pipx --no-cache-dir && \ + pipx install virtualenv && \ + pipx install pipenv && \ + pipx install poetry==1.1.12 && \ + pipx install awscli==1.* && \ + pipx install aws-lambda-builders==1.2.0 && \ + pipx install aws-sam-cli==1.15.0 diff --git a/runtimes/python3.10-arm64/build/build.sh b/runtimes/python3.10-arm64/build/build.sh new file mode 100755 index 0000000..657a285 --- /dev/null +++ b/runtimes/python3.10-arm64/build/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euxo pipefail + +docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-build-arm64 . diff --git a/runtimes/python3.10-arm64/build/publish.sh b/runtimes/python3.10-arm64/build/publish.sh new file mode 100755 index 0000000..5e3c611 --- /dev/null +++ b/runtimes/python3.10-arm64/build/publish.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euxo pipefail + +export PUBLISH_DATE=$(date "+%Y%m%d%H%M") +export NO_ARCH_TAG="mlupin/docker-lambda:python3.10-build" +export BASE_IMAGE="${NO_ARCH_TAG}-arm64" + +docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE} +docker push ${BASE_IMAGE} +docker push ${BASE_IMAGE}-${PUBLISH_DATE} diff --git a/runtimes/python3.10-arm64/run/Dockerfile b/runtimes/python3.10-arm64/run/Dockerfile new file mode 100644 index 0000000..5c90b7e --- /dev/null +++ b/runtimes/python3.10-arm64/run/Dockerfile @@ -0,0 +1,21 @@ +FROM arm64v8/amazonlinux:2 + +RUN yum install -y tar gzip + +RUN curl https://docker-lambda.s3.amazonaws.com/fs/python3.10-arm64.tgz | tar -zx -C /opt + +FROM mlupin/docker-lambda:provided.al2-arm64 +FROM mlupin/docker-lambda:base-arm64 + +ENV PATH=/var/lang/bin:$PATH \ + LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \ + AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \ + AWS_EXECUTION_ARCH=arm64 + +COPY --from=0 /opt/* /var/ + +COPY --from=1 /var/runtime/init /var/rapid/init + +USER sbx_user1051 + +ENTRYPOINT ["/var/rapid/init", "--bootstrap", "/var/runtime/bootstrap", "--enable-msg-logs"] diff --git a/runtimes/python3.10-arm64/run/build.sh b/runtimes/python3.10-arm64/run/build.sh new file mode 100755 index 0000000..ee41412 --- /dev/null +++ b/runtimes/python3.10-arm64/run/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euxo pipefail + +docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-arm64 . diff --git a/runtimes/python3.10-arm64/run/publish.sh b/runtimes/python3.10-arm64/run/publish.sh new file mode 100755 index 0000000..7cb61c7 --- /dev/null +++ b/runtimes/python3.10-arm64/run/publish.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euxo pipefail + +export PUBLISH_DATE=$(date "+%Y%m%d%H%M") +export NO_ARCH_TAG="mlupin/docker-lambda:python3.10" +export BASE_IMAGE="${NO_ARCH_TAG}-arm64" + +docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE} +docker push ${BASE_IMAGE} +docker push ${BASE_IMAGE}-${PUBLISH_DATE} diff --git a/runtimes/python3.10-x86_64/build/Dockerfile b/runtimes/python3.10-x86_64/build/Dockerfile new file mode 100644 index 0000000..7703389 --- /dev/null +++ b/runtimes/python3.10-x86_64/build/Dockerfile @@ -0,0 +1,25 @@ +FROM mlupin/docker-lambda:python3.10-x86_64 + +FROM mlupin/docker-lambda:build-x86_64 + +ENV PATH=/var/lang/bin:$PATH \ + LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \ + AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \ + AWS_EXECUTION_ARCH=x86_64 \ + PKG_CONFIG_PATH=/var/lang/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/share/pkgconfig \ + PIPX_BIN_DIR=/var/lang/bin \ + PIPX_HOME=/var/lang/pipx + +COPY --from=0 /var/runtime /var/runtime +COPY --from=0 /var/lang /var/lang +COPY --from=0 /var/rapid /var/rapid + +# Add these as a separate layer as they get updated frequently +RUN pip install -U pip setuptools wheel --no-cache-dir && \ + pip install pipx --no-cache-dir && \ + pipx install virtualenv && \ + pipx install pipenv && \ + pipx install poetry==1.1.12 && \ + pipx install awscli==1.* && \ + pipx install aws-lambda-builders==1.2.0 && \ + pipx install aws-sam-cli==1.15.0 diff --git a/runtimes/python3.10-x86_64/build/build.sh b/runtimes/python3.10-x86_64/build/build.sh new file mode 100755 index 0000000..f05f374 --- /dev/null +++ b/runtimes/python3.10-x86_64/build/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euxo pipefail + +docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-build-x86_64 . diff --git a/runtimes/python3.10-x86_64/build/publish.sh b/runtimes/python3.10-x86_64/build/publish.sh new file mode 100755 index 0000000..a7a15c4 --- /dev/null +++ b/runtimes/python3.10-x86_64/build/publish.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euxo pipefail + +export PUBLISH_DATE=$(date "+%Y%m%d%H%M") +export NO_ARCH_TAG="mlupin/docker-lambda:python3.10-build" +export BASE_IMAGE="${NO_ARCH_TAG}-x86_64" + +docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE} +docker push ${BASE_IMAGE} +docker push ${BASE_IMAGE}-${PUBLISH_DATE} diff --git a/runtimes/python3.10-x86_64/run/Dockerfile b/runtimes/python3.10-x86_64/run/Dockerfile new file mode 100644 index 0000000..a7363d6 --- /dev/null +++ b/runtimes/python3.10-x86_64/run/Dockerfile @@ -0,0 +1,21 @@ +FROM amd64/amazonlinux:2 + +RUN yum install -y tar gzip + +RUN curl https://docker-lambda.s3.amazonaws.com/fs/python3.10-x86_64.tgz | tar -zx -C /opt + +FROM mlupin/docker-lambda:provided.al2-x86_64 +FROM mlupin/docker-lambda:base-x86_64 + +ENV PATH=/var/lang/bin:$PATH \ + LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \ + AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \ + AWS_EXECUTION_ARCH=x86_64 + +COPY --from=0 /opt/* /var/ + +COPY --from=1 /var/runtime/init /var/rapid/init + +USER sbx_user1051 + +ENTRYPOINT ["/var/rapid/init", "--bootstrap", "/var/runtime/bootstrap", "--enable-msg-logs"] diff --git a/runtimes/python3.10-x86_64/run/build.sh b/runtimes/python3.10-x86_64/run/build.sh new file mode 100755 index 0000000..4cbb4e0 --- /dev/null +++ b/runtimes/python3.10-x86_64/run/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euxo pipefail + +docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-x86_64 . diff --git a/runtimes/python3.10-x86_64/run/publish.sh b/runtimes/python3.10-x86_64/run/publish.sh new file mode 100755 index 0000000..b562f55 --- /dev/null +++ b/runtimes/python3.10-x86_64/run/publish.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euxo pipefail + +export PUBLISH_DATE=$(date "+%Y%m%d%H%M") +export NO_ARCH_TAG="mlupin/docker-lambda:python3.10" +export BASE_IMAGE="${NO_ARCH_TAG}-x86_64" + +docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE} +docker push ${BASE_IMAGE} +docker push ${BASE_IMAGE}-${PUBLISH_DATE}