From c08ae6c9560aa51b7ade361ce1358edafc05e62f Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 18:33:41 +0200 Subject: [PATCH 1/9] add copying files to/from container Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 41 ++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index e9415441..1a538903 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,7 +1,10 @@ import contextlib +import io +import tarfile +from pathlib import Path from platform import system from socket import socket -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Tuple import docker.errors from docker import version @@ -52,6 +55,7 @@ def __init__( self._network: Optional[Network] = None self._network_aliases: Optional[list[str]] = None self._kwargs = kwargs + self._files: list[Tuple[Path, Path]] = [] def with_env(self, key: str, value: str) -> Self: self.env[key] = value @@ -78,6 +82,33 @@ def with_kwargs(self, **kwargs) -> Self: self._kwargs = kwargs return self + def with_copy_file_to_container(self, source_file: Path, destination_file: Path) -> Self: + self._files.append((source_file, destination_file)) + + return self + + def copy_file_from_container(self, container_file: str, destination_file: str) -> str: + tar_stream, _ = self._container.get_archive(container_file) + + for chunk in tar_stream: + with tarfile.open(fileobj=io.BytesIO(chunk)) as tar: + for member in tar.getmembers(): + with open(destination_file, 'wb') as f: + f.write(tar.extractfile(member).read()) + + return destination_file + + @staticmethod + def _put_file_in_container(container, source_file: Path, destination_file: str): + data = io.BytesIO() + + with tarfile.open(fileobj=data, mode='w') as tar: + tar.add(source_file, arcname=destination_file) + + data.seek(0) + + container.put_archive("/", data) + def maybe_emulate_amd64(self) -> Self: if is_arm(): return self.with_kwargs(platform="linux/amd64") @@ -115,6 +146,14 @@ def start(self) -> Self: ) logger.info("Container started: %s", self._container.short_id) + if self._network: + self._network.connect(self._container.id, self._network_aliases) + + for file in self._files: + source, destination = file[0], file[1] + + DockerContainer._put_file_in_container(self._container, source, destination) + return self def stop(self, force=True, delete_volume=True) -> None: From 8ed3df03a763e3340853d9547a4becfdc497595c Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 18:35:44 +0200 Subject: [PATCH 2/9] rename vars Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 1a538903..f3525792 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -149,8 +149,8 @@ def start(self) -> Self: if self._network: self._network.connect(self._container.id, self._network_aliases) - for file in self._files: - source, destination = file[0], file[1] + for copy_spec in self._files: + source, destination = copy_spec[0], copy_spec[1] DockerContainer._put_file_in_container(self._container, source, destination) From e7b7990cb099f69df38af7d997d0d344109ec983 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 18:35:58 +0200 Subject: [PATCH 3/9] fix typing Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index f3525792..8c4d79a5 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -99,7 +99,7 @@ def copy_file_from_container(self, container_file: str, destination_file: str) - return destination_file @staticmethod - def _put_file_in_container(container, source_file: Path, destination_file: str): + def _put_file_in_container(container, source_file: Path, destination_file: Path): data = io.BytesIO() with tarfile.open(fileobj=data, mode='w') as tar: From 3040d2b2d22d2a5d3984edf35deca776a8fa6f45 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 18:36:46 +0200 Subject: [PATCH 4/9] make linter happy Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 8c4d79a5..ac2b49a5 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -4,7 +4,7 @@ from pathlib import Path from platform import system from socket import socket -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional import docker.errors from docker import version @@ -55,7 +55,7 @@ def __init__( self._network: Optional[Network] = None self._network_aliases: Optional[list[str]] = None self._kwargs = kwargs - self._files: list[Tuple[Path, Path]] = [] + self._files: list[tuple[Path, Path]] = [] def with_env(self, key: str, value: str) -> Self: self.env[key] = value @@ -93,7 +93,7 @@ def copy_file_from_container(self, container_file: str, destination_file: str) - for chunk in tar_stream: with tarfile.open(fileobj=io.BytesIO(chunk)) as tar: for member in tar.getmembers(): - with open(destination_file, 'wb') as f: + with open(destination_file, "wb") as f: f.write(tar.extractfile(member).read()) return destination_file @@ -102,7 +102,7 @@ def copy_file_from_container(self, container_file: str, destination_file: str) - def _put_file_in_container(container, source_file: Path, destination_file: Path): data = io.BytesIO() - with tarfile.open(fileobj=data, mode='w') as tar: + with tarfile.open(fileobj=data, mode="w") as tar: tar.add(source_file, arcname=destination_file) data.seek(0) From acd19f5a5bd8a39a5815a2387ba8bdedba7bd591 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 18:38:39 +0200 Subject: [PATCH 5/9] unify args using Path Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index ac2b49a5..1fb2a9ef 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -87,7 +87,7 @@ def with_copy_file_to_container(self, source_file: Path, destination_file: Path) return self - def copy_file_from_container(self, container_file: str, destination_file: str) -> str: + def copy_file_from_container(self, container_file: Path, destination_file: Path) -> Path: tar_stream, _ = self._container.get_archive(container_file) for chunk in tar_stream: From 40fd1101a6ac42cfc09511a2ab4873e6d6ca41ff Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Mon, 12 Aug 2024 22:27:50 +0200 Subject: [PATCH 6/9] refactor for transferable Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 23 ++++++++--------- core/testcontainers/core/transferable.py | 33 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 core/testcontainers/core/transferable.py diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 1fb2a9ef..6deb0e16 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -1,7 +1,7 @@ import contextlib import io +import os import tarfile -from pathlib import Path from platform import system from socket import socket from typing import TYPE_CHECKING, Optional @@ -16,6 +16,7 @@ from testcontainers.core.exceptions import ContainerStartException from testcontainers.core.labels import LABEL_SESSION_ID, SESSION_ID from testcontainers.core.network import Network +from testcontainers.core.transferable import Transferable from testcontainers.core.utils import inside_container, is_arm, setup_logger from testcontainers.core.waiting_utils import wait_container_is_ready, wait_for_logs @@ -55,7 +56,7 @@ def __init__( self._network: Optional[Network] = None self._network_aliases: Optional[list[str]] = None self._kwargs = kwargs - self._files: list[tuple[Path, Path]] = [] + self._files: list[Transferable] = [] def with_env(self, key: str, value: str) -> Self: self.env[key] = value @@ -82,12 +83,12 @@ def with_kwargs(self, **kwargs) -> Self: self._kwargs = kwargs return self - def with_copy_file_to_container(self, source_file: Path, destination_file: Path) -> Self: - self._files.append((source_file, destination_file)) + def with_copy_file_to_container(self, transferable: Transferable) -> Self: + self._files.append(transferable) return self - def copy_file_from_container(self, container_file: Path, destination_file: Path) -> Path: + def copy_file_from_container(self, container_file: os.PathLike, destination_file: os.PathLike) -> os.PathLike: tar_stream, _ = self._container.get_archive(container_file) for chunk in tar_stream: @@ -99,11 +100,11 @@ def copy_file_from_container(self, container_file: Path, destination_file: Path) return destination_file @staticmethod - def _put_file_in_container(container, source_file: Path, destination_file: Path): + def _put_data_in_container(container, transferable: Transferable): data = io.BytesIO() - with tarfile.open(fileobj=data, mode="w") as tar: - tar.add(source_file, arcname=destination_file) + with transferable as f, tarfile.open(fileobj=data, mode="w") as tar: + tar.add(f.input_path, arcname=f.output_path) data.seek(0) @@ -149,10 +150,8 @@ def start(self) -> Self: if self._network: self._network.connect(self._container.id, self._network_aliases) - for copy_spec in self._files: - source, destination = copy_spec[0], copy_spec[1] - - DockerContainer._put_file_in_container(self._container, source, destination) + for transferable in self._files: + DockerContainer._put_data_in_container(self._container, transferable) return self diff --git a/core/testcontainers/core/transferable.py b/core/testcontainers/core/transferable.py new file mode 100644 index 00000000..6f6baab0 --- /dev/null +++ b/core/testcontainers/core/transferable.py @@ -0,0 +1,33 @@ +import os +import tempfile +from typing import Union + + +class Transferable: + def __init__(self, input_data: Union[os.PathLike, bytes], output_path: os.PathLike): + self._input = input_data + self._output_path = output_path + + self._tmp_file: bool = False + + def __enter__(self): + if isinstance(self._input, bytes): + tmp_file = tempfile.NamedTemporaryFile(delete=False) + tmp_file.write(self._input) + + self._input = tmp_file.name + self._tmp_file = True + + return self + + def __exit__(self, *args): + if self._tmp_file: + os.remove(self._input) + + @property + def input_path(self): + return self._input + + @property + def output_path(self): + return self._output_path From a2f60940282871f49d8133a241fe61017fee73a3 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Tue, 13 Aug 2024 08:13:47 +0200 Subject: [PATCH 7/9] add tests Signed-off-by: mgorsk1 --- core/tests/test_core.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/tests/test_core.py b/core/tests/test_core.py index 8d0c7794..95c5a406 100644 --- a/core/tests/test_core.py +++ b/core/tests/test_core.py @@ -8,6 +8,7 @@ from testcontainers.core.container import DockerContainer from testcontainers.core.image import DockerImage +from testcontainers.core.transferable import Transferable from testcontainers.core.waiting_utils import wait_for_logs @@ -92,3 +93,33 @@ def test_docker_image_with_custom_dockerfile_path(dockerfile_path: Optional[Path with DockerContainer(str(image)) as container: assert container._container.image.short_id.endswith(image_short_id), "Image ID mismatch" assert container.get_logs() == (("Hello world!\n").encode(), b""), "Container logs mismatch" + + +def test_docker_start_with_copy_file_to_container_from_binary_transferable(tmp_path: Path) -> None: + container = DockerContainer("nginx") + data = "test_docker_start_with_copy_file_to_container_from_binary_transferable" + + input_data = data.encode("utf-8") + output_file = Path("/tmp/test_docker_start_with_copy_file_to_container_from_binary_transferable.txt") + + container.with_copy_file_to_container(Transferable(input_data, output_file)).start() + + _, stdout = container.exec(f"cat {output_file}") + assert stdout.decode() == data + + +def test_docker_start_with_copy_file_to_container_from_file_transferable(tmp_path: Path) -> None: + container = DockerContainer("nginx") + data = "test_docker_start_with_copy_file_to_container_from_file_transferable" + + with tempfile.NamedTemporaryFile(delete=True) as f: + f.write(data.encode("utf-8")) + f.seek(0) + + input_file = Path(f.name) + output_file = Path("/tmp/test_docker_start_with_copy_file_to_container_from_file_transferable.txt") + + container.with_copy_file_to_container(Transferable(input_file, output_file)).start() + + _, stdout = container.exec(f"cat {output_file}") + assert stdout.decode() == data From fc400160d62d77f14e4a8f1bc85eb77f20ca6753 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Tue, 13 Aug 2024 13:10:50 +0200 Subject: [PATCH 8/9] remove old network approach Signed-off-by: mgorsk1 --- core/testcontainers/core/container.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/testcontainers/core/container.py b/core/testcontainers/core/container.py index 6deb0e16..a90c726b 100644 --- a/core/testcontainers/core/container.py +++ b/core/testcontainers/core/container.py @@ -147,8 +147,6 @@ def start(self) -> Self: ) logger.info("Container started: %s", self._container.short_id) - if self._network: - self._network.connect(self._container.id, self._network_aliases) for transferable in self._files: DockerContainer._put_data_in_container(self._container, transferable) From 6f25274eafbaefe31cc5139f0e10dbb0974848f8 Mon Sep 17 00:00:00 2001 From: mgorsk1 Date: Tue, 13 Aug 2024 14:27:21 +0200 Subject: [PATCH 9/9] remove tmppath Signed-off-by: mgorsk1 --- core/tests/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tests/test_core.py b/core/tests/test_core.py index 95c5a406..0329f975 100644 --- a/core/tests/test_core.py +++ b/core/tests/test_core.py @@ -95,7 +95,7 @@ def test_docker_image_with_custom_dockerfile_path(dockerfile_path: Optional[Path assert container.get_logs() == (("Hello world!\n").encode(), b""), "Container logs mismatch" -def test_docker_start_with_copy_file_to_container_from_binary_transferable(tmp_path: Path) -> None: +def test_docker_start_with_copy_file_to_container_from_binary_transferable() -> None: container = DockerContainer("nginx") data = "test_docker_start_with_copy_file_to_container_from_binary_transferable" @@ -108,7 +108,7 @@ def test_docker_start_with_copy_file_to_container_from_binary_transferable(tmp_p assert stdout.decode() == data -def test_docker_start_with_copy_file_to_container_from_file_transferable(tmp_path: Path) -> None: +def test_docker_start_with_copy_file_to_container_from_file_transferable() -> None: container = DockerContainer("nginx") data = "test_docker_start_with_copy_file_to_container_from_file_transferable"