Skip to content

Commit 3450b2d

Browse files
jleanizjleaniz
authored
Manage dependencies using Poetry (google#1348)
* Added Poetry configuration file and lock file * Updates to package management * Updates to Dockerfiles * Updated comments and dependencies * Cleanup old changes * Update requirements text files * Minor updates * Add skeleton setup.py for legacy compatibility test single stage dockerfilre * Updates to dockerfile * Updates to dockerfiles and scripts * Add poetry.toml file * Update dependencies * Update dockerfiles * Update dockerfiles * Update gh actions * Update dockerfile * Updates to docker-compose and dockerfile * Updates to docker files * Update docker file * Updates to docker file * Update dockerfiles * Update dockerfiles * Update poetry lock and dockerfiles * Add dockerignore files * Update dockerignore files * Update dockerignore files * Update dockerignore file * Update dockerignore file * Updates to dockerfiles and start script * Update legacy requirements files * Update dockerignore files and versioning * Add Ramses to authors file * Updates to client project files * Updates to poetry files * Updates * Updates * Update pyproject.toml for api-lib * review comments * Update documentation * fix typo in docs
1 parent 2a1cbff commit 3450b2d

40 files changed

+10715
-355
lines changed

‎.github/workflows/actions.yml

+9-16
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,17 @@ jobs:
1818
- name: Pull latest docker image for cache
1919
run: |
2020
docker pull ${{ matrix.docker_base_image }}
21-
docker pull us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker-dev:latest
22-
- name: Build Turbinia worker docker image
23-
run: docker build --build-arg PPA_TRACK=${{ matrix.gift_ppa_track }} --cache-from=${{ matrix.docker_base_image }},us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker-dev:latest -t turbinia-worker-dev -f docker/worker/Dockerfile .
24-
- name: Run test (turbinia-worker) container
21+
docker pull us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-unit-tests:latest
22+
- name: Build Turbinia unit test docker image
23+
run: docker build --build-arg PPA_TRACK=${{ matrix.gift_ppa_track }} --cache-from=${{ matrix.docker_base_image }},us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-unit-tests:latest -t turbinia-unit-tests -f docker/tests/Dockerfile .
24+
- name: Run test (turbinia-unit-tests) container
2525
run: |
26-
docker run --name turbinia-worker --entrypoint "/bin/bash" -it -d -t turbinia-worker-dev:latest
26+
docker run --name turbinia-unit-tests --entrypoint "/bin/bash" -it -d -t turbinia-unit-tests:latest
2727
- name: Configure python3
2828
run: |
29-
docker exec -u root -t turbinia-worker bash -c "update-alternatives --install /usr/bin/python python /usr/bin/python3 1"
30-
docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -V"
31-
- name: Install and configure test dependencies
32-
run: |
33-
docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet --upgrade pip"
34-
docker exec -u root -t turbinia-worker bash -c "cd /tmp/turbinia/api/client && /usr/bin/python -m pip install --quiet ."
35-
docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet -r /tmp/turbinia/api/cli/requirements.txt"
36-
docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet mock nose coverage yapf"
37-
docker exec -u root -t turbinia-worker bash -c "/usr/bin/python -m pip install --quiet tox"
29+
docker exec -u root -t turbinia-unit-tests bash -c "update-alternatives --install /usr/bin/python python /usr/bin/python3 1"
30+
docker exec -u root -t turbinia-unit-tests bash -c "/usr/bin/python -V"
3831
- name: Run Tests
3932
run: |
40-
docker exec -u root -t turbinia-worker bash -c "cd /tmp && ./run_tests.py"
41-
docker exec -u root -t turbinia-worker bash -c "cd /tmp && tox --sitepackages ${TOXENV}"
33+
docker exec -u root -t turbinia-unit-tests bash -c "poetry run ./run_tests.py"
34+
docker exec -u root -t turbinia-unit-tests bash -c "tox --sitepackages ${TOXENV}"

‎.github/workflows/e2e.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,4 @@ jobs:
4040
- name: Run E2E test
4141
run: |
4242
chmod +x ./turbinia/e2e/e2e-local.sh
43-
./turbinia/e2e/e2e-local.sh
43+
./turbinia/e2e/e2e-local.sh

‎.github/workflows/publish-pypi.yml

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
jobs:
8+
build:
9+
name: Build distribution 📦
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- uses: actions/checkout@v4
14+
- name: Set up Python
15+
uses: actions/setup-python@v4
16+
with:
17+
python-version: "3.10"
18+
- name: Install pypa/build
19+
run: python3 -m pip install poetry wheel --user
20+
- name: Build a binary wheel and a source tarball
21+
run: python3 -m poetry build
22+
- name: Store the distribution packages
23+
uses: actions/upload-artifact@v3
24+
with:
25+
name: python-package-distributions
26+
path: dist/
27+
28+
publish-to-pypi:
29+
name: Publish Python 🐍 distribution 📦 to PyPI
30+
if: startsWith(github.ref, 'refs/tags/20') # only publish to PyPI on tag pushes
31+
needs:
32+
- build
33+
runs-on: ubuntu-latest
34+
environment:
35+
name: pypi
36+
url: https://pypi.org/p/turbinia
37+
permissions:
38+
id-token: write # IMPORTANT: mandatory for trusted publishing
39+
40+
steps:
41+
- name: Download all the dists
42+
uses: actions/download-artifact@v3
43+
with:
44+
name: python-package-distributions
45+
path: dist/
46+
- name: Publish distribution 📦 to PyPI
47+
uses: pypa/gh-action-pypi-publish@release/v1
48+
49+
github-release:
50+
name: >-
51+
Sign the Python 🐍 distribution 📦 with Sigstore
52+
and upload them to GitHub Release
53+
needs:
54+
- publish-to-pypi
55+
runs-on: ubuntu-latest
56+
57+
permissions:
58+
contents: write # IMPORTANT: mandatory for making GitHub Releases
59+
id-token: write # IMPORTANT: mandatory for sigstore
60+
61+
steps:
62+
- name: Download all the dists
63+
uses: actions/download-artifact@v3
64+
with:
65+
name: python-package-distributions
66+
path: dist/
67+
- name: Sign the dists with Sigstore
68+
uses: sigstore/[email protected]
69+
with:
70+
inputs: >-
71+
./dist/*.tar.gz
72+
./dist/*.whl
73+
- name: Upload artifact signatures to GitHub Release
74+
# Te release must already be created or this will fail.
75+
env:
76+
GITHUB_TOKEN: ${{ github.token }}
77+
# Upload to GitHub Release using the `gh` CLI.
78+
# `dist/` contains the built packages, and the
79+
# sigstore-produced signatures and certificates.
80+
run: >-
81+
gh release upload
82+
'${{ github.ref_name }}' dist/**
83+
--repo '${{ github.repository }}'
84+
85+
publish-to-testpypi:
86+
name: Publish Python 🐍 distribution 📦 to TestPyPI
87+
needs:
88+
- build
89+
runs-on: ubuntu-latest
90+
91+
environment:
92+
name: testpypi
93+
url: https://test.pypi.org/p/turbinia
94+
95+
permissions:
96+
id-token: write # IMPORTANT: mandatory for trusted publishing
97+
98+
steps:
99+
- name: Download all the dists
100+
uses: actions/download-artifact@v3
101+
with:
102+
name: python-package-distributions
103+
path: dist/
104+
- name: Publish distribution 📦 to TestPyPI
105+
uses: pypa/gh-action-pypi-publish@release/v1
106+
with:
107+
repository-url: https://test.pypi.org/legacy/

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,4 @@ venv/
4141
.venv/
4242

4343
# Ignore OpenAPI code generator temporary files
44-
.openapi-generator/
44+
.openapi-generator/

‎AUTHORS

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ Turbinia is developed by (in alphabetical order):
55

66
* Google Inc. (*@google.com)
77
* Aaron Peterson ([email protected])
8-
* Wajih Yassine ([email protected])
98
* Johan Berggren ([email protected])
9+
* Juan Leaniz ([email protected])
10+
* Ramses de Beer ([email protected])
11+
* Wajih Yassine ([email protected])
1012

1113
To reach the authors, please use the Turbinia development mailing
1214
list <[email protected]>.

‎dfvfs_requirements.txt

-32
This file was deleted.

‎docker/api_server/.dockerignore

-2
This file was deleted.

‎docker/api_server/Dockerfile

+27-20
Original file line numberDiff line numberDiff line change
@@ -7,38 +7,45 @@ COPY web/. .
77
RUN npm run build
88

99
# Build Turbinia API Server, copy from build, and setup rest of requirements
10-
FROM ubuntu:22.04
10+
FROM ubuntu:22.04 as build-stage2
11+
12+
ENV DEBIAN_FRONTEND=noninteractive \
13+
PIP_NO_CACHE_DIR=1
1114

12-
ENV DEBIAN_FRONTEND=noninteractive
1315
COPY --from=build-stage /tmp/web/dist /web/dist
1416
RUN apt-get update && apt-get -y upgrade
15-
RUN apt-get -y install python3-pip git wget supervisor curl
16-
17+
RUN apt-get -y install git python3-pip
1718
RUN pip3 install pip --upgrade
18-
RUN pip3 install urllib3 cryptography requests --upgrade
19-
20-
ADD requirements.txt /tmp/
21-
RUN cd /tmp/ && pip3 install -r requirements.txt
22-
23-
ADD . /tmp/
24-
25-
# unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
26-
RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi
27-
28-
RUN cd /tmp/ && python3 setup.py install
29-
30-
RUN useradd -r -s /bin/nologin -u 999 turbinia
19+
RUN pip3 install poetry
3120

21+
RUN useradd -r -s /sbin/nologin -u 999 turbinia
3222
RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \
3323
&& mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \
3424
&& mkdir -p /etc/turbinia/ \
3525
&& chown -R turbinia:turbinia /var/lib/turbinia/ \
3626
&& chown -R turbinia:turbinia /etc/turbinia/ \
37-
&& chown -R turbinia:turbinia /var/log/turbinia/
27+
&& chown -R turbinia:turbinia /var/log/turbinia/ \
28+
&& mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia
3829

39-
COPY docker/api_server/start.sh /home/turbinia/start.sh
40-
RUN chmod +rwx /home/turbinia/start.sh
30+
# Drop privileges and set the working directory
4131
USER turbinia
32+
WORKDIR /home/turbinia
33+
34+
# Copy requirements and install dependencies to cache them in docker layer
35+
COPY --chown=turbinia:turbinia ./pyproject.toml ./poetry.toml ./poetry.lock /home/turbinia/
36+
RUN poetry install --no-interaction --no-ansi --no-root
37+
38+
ENV PATH="/home/turbinia/.venv/bin:$PATH" \
39+
VIRTUAL_ENV=/home/turbinia/.venv
40+
41+
# Copy the source directory to the container
42+
COPY --chown=turbinia:turbinia . /home/turbinia/
43+
COPY --chown=turbinia:turbinia docker/api_server/start.sh /home/turbinia/start.sh
44+
RUN chmod +rwx /home/turbinia/start.sh
45+
46+
# Install Turbinia package -- will skip dependencies if installed
47+
RUN poetry install --no-interaction --no-ansi
48+
4249
CMD ["/home/turbinia/start.sh"]
4350
# Expose Prometheus and API endpoints.
4451
EXPOSE 9200/tcp

‎docker/api_server/Dockerfile.dockerignore

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
nodemodules
2+
dist
3+
.git
4+
.venv
5+
.vscode
6+
test_data
7+
.github
8+
.devcontainer
9+
docs
10+
config
11+
build
12+
*.pyc
13+
*.pyo
14+
*.pyd
15+
__pycache__
16+
.cache
17+
turbinia.egg-info

‎docker/api_server/start.sh

+6-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,12 @@ then
77
fi
88

99
# Start Turbinia API server
10-
/usr/local/bin/turbiniactl api_server
10+
if [ ! -z ${TURBINIA_LOG_FILE+x} ]
11+
then
12+
poetry run turbiniactl $TURBINIA_EXTRA_ARGS -L $TURBINIA_LOG_FILE api_server
13+
else
14+
poetry run turbiniactl $TURBINIA_EXTRA_ARGS api_server
15+
fi
1116

1217
# Don't exit
1318
while sleep 1000; do :; done

‎docker/local/docker-compose.yml

+13-12
Original file line numberDiff line numberDiff line change
@@ -57,17 +57,18 @@ services:
5757
- LC_ALL=C.UTF-8
5858
- LANG=C.UTF-8
5959
- TURBINIA_EXTRA_ARGS=${TURBINIA_EXTRA_ARGS}
60-
# Uncomment below in case you want to run a second worker on the same host.
61-
# turbinia-worker2:
62-
# #image: "turbinia-worker-dev" # Use this for local development and comment out below line
63-
# image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker:latest" # Latest stable
64-
# container_name: turbinia-worker2
65-
# privileged: true
6660

67-
# volumes:
68-
# - $PWD/evidence:/evidence
69-
# - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
61+
# Uncomment below in case you want to run a second worker on the same host.
62+
# turbinia-worker2:
63+
# image: "turbinia-worker-dev" # Use this for local development and comment out below line
64+
# image: "us-docker.pkg.dev/osdfir-registry/turbinia/release/turbinia-worker:latest" # Latest stable
65+
# container_name: turbinia-worker2
66+
# privileged: true
7067

71-
# environment:
72-
# - LC_ALL=C.UTF-8
73-
# - LANG=C.UTF-8
68+
# volumes:
69+
# - $PWD/evidence:/evidence
70+
# - $PWD/conf/turbinia.conf:/etc/turbinia/turbinia.conf
71+
72+
# environment:
73+
# - LC_ALL=C.UTF-8
74+
# - LANG=C.UTF-8

‎docker/server/Dockerfile

+23-13
Original file line numberDiff line numberDiff line change
@@ -13,28 +13,38 @@ RUN apt-get update && apt-get -y upgrade && apt-get -y install \
1313
sudo \
1414
&& apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*
1515

16-
RUN pip3 install pip --upgrade \
17-
&& pip3 install urllib3 cryptography requests --upgrade
16+
RUN pip3 install pip --upgrade
17+
RUN pip3 install poetry
1818

19-
ADD . /tmp/
20-
RUN cd /tmp/ && pip3 install -r requirements.txt
21-
22-
# unshallow and fetch all tags so our build systems pickup the correct git tag if it's a shallow clone
23-
RUN if $(cd /tmp/ && git rev-parse --is-shallow-repository); then cd /tmp/ && git fetch --prune --unshallow && git fetch --depth=1 origin +refs/tags/*:refs/tags/*; fi \
24-
&& cd /tmp/ && python3 setup.py install
25-
26-
RUN useradd -r -s /bin/nologin -u 999 turbinia
19+
RUN useradd -r -s /sbin/nologin -u 999 turbinia
2720

2821
RUN mkdir /etc/turbinia && mkdir -p /mnt/turbinia/ && mkdir -p /var/lib/turbinia/ \
2922
&& mkdir -p /var/log/turbinia/ && chown -R turbinia:turbinia /mnt/turbinia/ \
3023
&& mkdir -p /etc/turbinia/ \
3124
&& chown -R turbinia:turbinia /var/lib/turbinia/ \
3225
&& chown -R turbinia:turbinia /etc/turbinia/ \
33-
&& chown -R turbinia:turbinia /var/log/turbinia/
26+
&& chown -R turbinia:turbinia /var/log/turbinia/ \
27+
&& mkdir -p /home/turbinia && chown -R turbinia:turbinia /home/turbinia
3428

35-
COPY docker/server/start.sh /home/turbinia/start.sh
36-
RUN chmod +rwx /home/turbinia/start.sh
29+
# Drop privileges and set the working directory
3730
USER turbinia
31+
WORKDIR /home/turbinia
32+
33+
# Copy requirements and install dependencies to cache them in docker layer
34+
COPY --chown=turbinia:turbinia ./pyproject.toml ./poetry.toml ./poetry.lock /home/turbinia/
35+
RUN poetry install --no-interaction --no-ansi --no-root
36+
37+
ENV PATH="/home/turbinia/.venv/bin:$PATH" \
38+
VIRTUAL_ENV=/home/turbinia/.venv
39+
40+
# Copy the source directory to the container
41+
COPY --chown=turbinia:turbinia . /home/turbinia/
42+
COPY --chown=turbinia:turbinia docker/server/start.sh /home/turbinia/start.sh
43+
RUN chmod +rwx /home/turbinia/start.sh
44+
45+
# Install Turbinia package -- will skip dependencies if installed
46+
RUN poetry install --no-interaction --no-ansi
47+
3848
CMD ["/home/turbinia/start.sh"]
3949
# Expose Prometheus endpoint.
4050
EXPOSE 9200/tcp

0 commit comments

Comments
 (0)