Add Python 3.10 runtime

README.md

@@ -34,6 +34,7 @@ the [AWS CLI](https://aws.amazon.com/cli/).
     - [nodejs18.x](#nodejs18x)
     - [python3.8](#python38)
     - [python3.9](#python39)
+    - [python3.10](#python310)
     - [ruby2.7](#ruby27)
     - [java8.al2](#java8al2)
     - [java11](#java11)
@@ -166,6 +167,24 @@ Build images
 | x86_64    | `mlupin/docker-lambda:python3.9-build-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.9-build-x86_64` |
 | arm64     | `mlupin/docker-lambda:python3.9-build-arm64`  | `ghcr.io/mlupine/docker-lambda:python3.9-build-arm64`  |
 
+### python3.10
+
+Runtime images
+
+| Platform  | Docker Hub                               | GitHub Container Registry                         |
+| --------- | ---------------------------------------- | ------------------------------------------------- |
+| Universal | `mlupin/docker-lambda:python3.10`        | `ghcr.io/mlupine/docker-lambda:python3.10`        |
+| x86_64    | `mlupin/docker-lambda:python3.10-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.10-x86_64` |
+| arm64     | `mlupin/docker-lambda:python3.10-arm64`  | `ghcr.io/mlupine/docker-lambda:python3.10-arm64`  |
+
+Build images
+
+| Platform  | Docker Hub                                     | GitHub Container Registry                               |
+| --------- | ---------------------------------------------- | ------------------------------------------------------- |
+| Universal | `mlupin/docker-lambda:python3.10-build`        | `ghcr.io/mlupine/docker-lambda:python3.10-build`        |
+| x86_64    | `mlupin/docker-lambda:python3.10-build-x86_64` | `ghcr.io/mlupine/docker-lambda:python3.10-build-x86_64` |
+| arm64     | `mlupin/docker-lambda:python3.10-build-arm64`  | `ghcr.io/mlupine/docker-lambda:python3.10-build-arm64`  |
+
 ### ruby2.7
 
 Runtime images

dump-fs/dump-python310/handler.py

@@ -0,0 +1,50 @@
+from __future__ import print_function
+
+import os
+import sys
+import subprocess
+import json
+import boto3
+from boto3.s3.transfer import S3Transfer
+
+TRANSFER = S3Transfer(boto3.client('s3'))
+
+def lambda_handler_arm64(event, context):
+    return lambda_handler(event, context, "arm64")
+
+def lambda_handler_x86_64(event, context):
+    return lambda_handler(event, context, "x86_64")
+
+def lambda_handler(event, context, arch):
+    if 'cmd' in event:
+        return print(subprocess.check_output(['sh', '-c', event['cmd']]).decode('utf-8'))
+
+    filename = f'python3.10-{arch}.tgz'
+
+    subprocess.call(['touch', f'/tmp/{filename}'])
+
+    subprocess.call(['sh', '-c', f'tar -cpzf /tmp/{filename} --numeric-owner --ignore-failed-read ' +
+                     '/var/runtime /var/lang /var/rapid'])
+
+    print('Zipping done! Uploading...')
+
+    TRANSFER.upload_file(f'/tmp/{filename}', 'docker-lambda',
+                         f'fs/{filename}', extra_args={'ACL': 'public-read'})
+
+    print('Uploading done!')
+
+    info = {'sys.executable': sys.executable,
+            'sys.argv': sys.argv,
+            'sys.path': sys.path,
+            'os.getcwd': os.getcwd(),
+            '__file__': __file__,
+            'os.environ': {k: str(v) for k, v in os.environ.items()},
+            'context': {k: str(v) for k, v in context.__dict__.items()},
+            'proc environ': subprocess.check_output(
+                ['sh', '-c', 'echo /proc/1/environ; xargs -n 1 -0 < /proc/1/environ']).decode('utf-8').splitlines(),
+            'ps aux': subprocess.check_output(
+                ['bash', '-O', 'extglob', '-c', 'for cmd in /proc/+([0-9])/cmdline; do echo $cmd; xargs -n 1 -0 < $cmd; done']).decode('utf-8').splitlines()}
+
+    print(json.dumps(info, indent=2))
+
+    return info

dump-fs/serverless.yml

@@ -94,6 +94,27 @@ functions:
     layers:
       - { Ref: TarARMLambdaLayer }
 
+  dump-python310-x86_64:
+    handler: dump-python310/handler.lambda_handler_x86_64
+    package:
+      patterns:
+        - "!**"
+        - "dump-python310/**"
+    runtime: python3.10
+    architecture: x86_64
+    layers:
+      - { Ref: TarX86LambdaLayer }
+  dump-python310-arm64:
+    handler: dump-python310/handler.lambda_handler_arm64
+    package:
+      patterns:
+        - "!**"
+        - "dump-python310/**"
+    runtime: python3.10
+    architecture: arm64
+    layers:
+      - { Ref: TarARMLambdaLayer }
+
   dump-ruby27-x86_64:
     handler: dump-ruby27/handler.lambda_handler_x86_64
     package:

examples/test-all.sh

@@ -18,10 +18,12 @@ npm test
 cd ${EXAMPLES_DIR}/python
 docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.8 lambda_function.lambda_handler
 docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.9 lambda_function.lambda_handler
+docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:python3.10 lambda_function.lambda_handler
 
 cd ${EXAMPLES_DIR}/python
 docker run --rm -it mlupin/docker-lambda:python3.8-build pip install marisa-trie
 docker run --rm -it mlupin/docker-lambda:python3.9-build pip install marisa-trie
+docker run --rm -it mlupin/docker-lambda:python3.10-build pip install marisa-trie
 
 cd ${EXAMPLES_DIR}/ruby
 docker run --rm -v "$PWD":/var/task mlupin/docker-lambda:ruby2.7 lambda_function.lambda_handler

runtimes/python3.10-arm64/build/Dockerfile

@@ -0,0 +1,25 @@
+FROM mlupin/docker-lambda:python3.10-arm64
+
+FROM mlupin/docker-lambda:build-arm64
+
+ENV PATH=/var/lang/bin:$PATH \
+  LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \
+  AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \
+  AWS_EXECUTION_ARCH=arm64 \
+  PKG_CONFIG_PATH=/var/lang/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/share/pkgconfig \
+  PIPX_BIN_DIR=/var/lang/bin \
+  PIPX_HOME=/var/lang/pipx
+
+COPY --from=0 /var/runtime /var/runtime
+COPY --from=0 /var/lang /var/lang
+COPY --from=0 /var/rapid /var/rapid
+
+# Add these as a separate layer as they get updated frequently
+RUN pip install -U pip setuptools wheel --no-cache-dir && \
+  pip install pipx --no-cache-dir && \
+  pipx install virtualenv && \
+  pipx install pipenv && \
+  pipx install poetry==1.1.12 && \
+  pipx install awscli==1.* && \
+  pipx install aws-lambda-builders==1.2.0 && \
+  pipx install aws-sam-cli==1.15.0

runtimes/python3.10-arm64/build/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-build-arm64 .

runtimes/python3.10-arm64/build/publish.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+export PUBLISH_DATE=$(date "+%Y%m%d%H%M")
+export NO_ARCH_TAG="mlupin/docker-lambda:python3.10-build"
+export BASE_IMAGE="${NO_ARCH_TAG}-arm64"
+
+docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE}
+docker push ${BASE_IMAGE}
+docker push ${BASE_IMAGE}-${PUBLISH_DATE}

runtimes/python3.10-arm64/run/Dockerfile

@@ -0,0 +1,21 @@
+FROM arm64v8/amazonlinux:2
+
+RUN yum install -y tar gzip
+
+RUN curl https://docker-lambda.s3.amazonaws.com/fs/python3.10-arm64.tgz | tar -zx -C /opt
+
+FROM mlupin/docker-lambda:provided.al2-arm64
+FROM mlupin/docker-lambda:base-arm64
+
+ENV PATH=/var/lang/bin:$PATH \
+    LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \
+    AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \
+    AWS_EXECUTION_ARCH=arm64
+
+COPY --from=0 /opt/* /var/
+
+COPY --from=1 /var/runtime/init /var/rapid/init
+
+USER sbx_user1051
+
+ENTRYPOINT ["/var/rapid/init", "--bootstrap", "/var/runtime/bootstrap", "--enable-msg-logs"]

runtimes/python3.10-arm64/run/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-arm64 .

runtimes/python3.10-arm64/run/publish.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+export PUBLISH_DATE=$(date "+%Y%m%d%H%M")
+export NO_ARCH_TAG="mlupin/docker-lambda:python3.10"
+export BASE_IMAGE="${NO_ARCH_TAG}-arm64"
+
+docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE}
+docker push ${BASE_IMAGE}
+docker push ${BASE_IMAGE}-${PUBLISH_DATE}

runtimes/python3.10-x86_64/build/Dockerfile

@@ -0,0 +1,25 @@
+FROM mlupin/docker-lambda:python3.10-x86_64
+
+FROM mlupin/docker-lambda:build-x86_64
+
+ENV PATH=/var/lang/bin:$PATH \
+  LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \
+  AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \
+  AWS_EXECUTION_ARCH=x86_64 \
+  PKG_CONFIG_PATH=/var/lang/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/share/pkgconfig \
+  PIPX_BIN_DIR=/var/lang/bin \
+  PIPX_HOME=/var/lang/pipx
+
+COPY --from=0 /var/runtime /var/runtime
+COPY --from=0 /var/lang /var/lang
+COPY --from=0 /var/rapid /var/rapid
+
+# Add these as a separate layer as they get updated frequently
+RUN pip install -U pip setuptools wheel --no-cache-dir && \
+  pip install pipx --no-cache-dir && \
+  pipx install virtualenv && \
+  pipx install pipenv && \
+  pipx install poetry==1.1.12 && \
+  pipx install awscli==1.* && \
+  pipx install aws-lambda-builders==1.2.0 && \
+  pipx install aws-sam-cli==1.15.0

runtimes/python3.10-x86_64/build/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-build-x86_64 .

runtimes/python3.10-x86_64/build/publish.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+export PUBLISH_DATE=$(date "+%Y%m%d%H%M")
+export NO_ARCH_TAG="mlupin/docker-lambda:python3.10-build"
+export BASE_IMAGE="${NO_ARCH_TAG}-x86_64"
+
+docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE}
+docker push ${BASE_IMAGE}
+docker push ${BASE_IMAGE}-${PUBLISH_DATE}

runtimes/python3.10-x86_64/run/Dockerfile

@@ -0,0 +1,21 @@
+FROM amd64/amazonlinux:2
+
+RUN yum install -y tar gzip
+
+RUN curl https://docker-lambda.s3.amazonaws.com/fs/python3.10-x86_64.tgz | tar -zx -C /opt
+
+FROM mlupin/docker-lambda:provided.al2-x86_64
+FROM mlupin/docker-lambda:base-x86_64
+
+ENV PATH=/var/lang/bin:$PATH \
+    LD_LIBRARY_PATH=/var/lang/lib:$LD_LIBRARY_PATH \
+    AWS_EXECUTION_ENV=AWS_Lambda_python3.10 \
+    AWS_EXECUTION_ARCH=x86_64
+
+COPY --from=0 /opt/* /var/
+
+COPY --from=1 /var/runtime/init /var/rapid/init
+
+USER sbx_user1051
+
+ENTRYPOINT ["/var/rapid/init", "--bootstrap", "/var/runtime/bootstrap", "--enable-msg-logs"]

runtimes/python3.10-x86_64/run/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+docker build --progress plain --squash -t mlupin/docker-lambda:python3.10-x86_64 .

runtimes/python3.10-x86_64/run/publish.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+export PUBLISH_DATE=$(date "+%Y%m%d%H%M")
+export NO_ARCH_TAG="mlupin/docker-lambda:python3.10"
+export BASE_IMAGE="${NO_ARCH_TAG}-x86_64"
+
+docker tag ${BASE_IMAGE} ${BASE_IMAGE}-${PUBLISH_DATE}
+docker push ${BASE_IMAGE}
+docker push ${BASE_IMAGE}-${PUBLISH_DATE}