From da39fb34416c5a301514a7b9da9a303dbc931a5d Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Sun, 21 Apr 2024 15:43:11 +0100 Subject: [PATCH 1/4] Add seed support in MySQL New capability of "seeding" a db container, by running a database-specific command or twelve to inject data (DB schema or sample data). Relies on the image's support for /docker-entrypoint-initdb.d/ folder --- .../mysql/testcontainers/mysql/__init__.py | 19 +++++++++++++++++++ modules/mysql/tests/seeds/01-schema.sql | 6 ++++++ modules/mysql/tests/seeds/02-seeds.sql | 3 +++ modules/mysql/tests/test_mysql.py | 13 +++++++++++++ 4 files changed, 41 insertions(+) create mode 100644 modules/mysql/tests/seeds/01-schema.sql create mode 100644 modules/mysql/tests/seeds/02-seeds.sql diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index 1b0751bc..970b6fde 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -40,6 +40,22 @@ class MySqlContainer(DbContainer): ... with engine.begin() as connection: ... result = connection.execute(sqlalchemy.text("select version()")) ... version, = result.fetchone() + + The optional :code:`seed` parameter enables arbitrary SQL files to be loaded. + This is perfect for schema and sample data. This works by mounting the seed to + `/docker-entrypoint-initdb./d`, which containerized MySQL are set up to load + automatically. + + .. doctest:: + >>> import sqlalchemy + >>> from testcontainers.mysql import MySqlContainer + >>> with MySqlContainer(seed="../../tests/") as mysql: + ... engine = sqlalchemy.create_engine(mysql.get_connection_url()) + ... with engine.begin() as connection: + ... query = "select * from stuff" # Can now rely on schema/data + ... result = connection.execute(sqlalchemy.text(query)) + ... first_stuff, = result.fetchone() + """ def __init__( @@ -50,6 +66,7 @@ def __init__( password: Optional[str] = None, dbname: Optional[str] = None, port: int = 3306, + seed: Optional[str] = None, **kwargs, ) -> None: raise_for_deprecated_parameter(kwargs, "MYSQL_USER", "username") @@ -67,6 +84,8 @@ def __init__( if self.username == "root": self.root_password = self.password + if seed is not None: + self.with_volume_mapping(seed, "/docker-entrypoint-initdb.d/") def _configure(self) -> None: self.with_env("MYSQL_ROOT_PASSWORD", self.root_password) diff --git a/modules/mysql/tests/seeds/01-schema.sql b/modules/mysql/tests/seeds/01-schema.sql new file mode 100644 index 00000000..ea398244 --- /dev/null +++ b/modules/mysql/tests/seeds/01-schema.sql @@ -0,0 +1,6 @@ +-- Sample SQL schema, no data +CREATE TABLE `stuff` ( + `id` mediumint NOT NULL AUTO_INCREMENT, + `name` VARCHAR(63) NOT NULL, + PRIMARY KEY (`id`) +); diff --git a/modules/mysql/tests/seeds/02-seeds.sql b/modules/mysql/tests/seeds/02-seeds.sql new file mode 100644 index 00000000..7ce78903 --- /dev/null +++ b/modules/mysql/tests/seeds/02-seeds.sql @@ -0,0 +1,3 @@ +-- Sample data, to be loaded after the schema +INSERT INTO stuff (name) +VALUES ("foo"), ("bar"), ("qux"), ("frob"); diff --git a/modules/mysql/tests/test_mysql.py b/modules/mysql/tests/test_mysql.py index ee1e2b45..847f99df 100644 --- a/modules/mysql/tests/test_mysql.py +++ b/modules/mysql/tests/test_mysql.py @@ -1,3 +1,4 @@ +from pathlib import Path import re from unittest import mock @@ -29,6 +30,18 @@ def test_docker_run_legacy_mysql(): assert row[0].startswith("5.7.44") +@pytest.mark.skipif(is_arm(), reason="mysql container not available for ARM") +def test_docker_run_mysql_8_seed(): + # Avoid pytest CWD path issues + SEEDS_PATH = (Path(__file__).parent / "seeds").absolute() + config = MySqlContainer("mysql:8", seed=SEEDS_PATH) + with config as mysql: + engine = sqlalchemy.create_engine(mysql.get_connection_url()) + with engine.begin() as connection: + result = connection.execute(sqlalchemy.text("select * from stuff")) + assert len(list(result)) == 4, "Should have gotten all the stuff" + + @pytest.mark.parametrize("version", ["11.3.2", "10.11.7"]) def test_docker_run_mariadb(version: str): with MySqlContainer(f"mariadb:{version}") as mariadb: From 713aa5931bbcdb7ade2fd88bfc410087a38996b7 Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Sun, 21 Apr 2024 16:22:55 +0100 Subject: [PATCH 2/4] Fix seed path in doctest --- modules/mysql/testcontainers/mysql/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index 970b6fde..e95b87d8 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -49,7 +49,7 @@ class MySqlContainer(DbContainer): .. doctest:: >>> import sqlalchemy >>> from testcontainers.mysql import MySqlContainer - >>> with MySqlContainer(seed="../../tests/") as mysql: + >>> with MySqlContainer(seed="../../tests/seeds/") as mysql: ... engine = sqlalchemy.create_engine(mysql.get_connection_url()) ... with engine.begin() as connection: ... query = "select * from stuff" # Can now rely on schema/data From ac0fc34717c27b53d3f291ca59b3218d2036c762 Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Fri, 26 Apr 2024 02:06:06 +0100 Subject: [PATCH 3/4] Experiment with transfering via tar archives --- core/testcontainers/core/generic.py | 4 ++++ modules/mysql/testcontainers/mysql/__init__.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index 515c2831..23c24ea7 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -71,7 +71,11 @@ def start(self) -> "DbContainer": self._configure() super().start() self._connect() + self._transfer_seed() return self def _configure(self) -> None: raise NotImplementedError + + def _transfer_seed(self) -> None: + pass diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index e95b87d8..c7ed97b7 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -11,7 +11,10 @@ # License for the specific language governing permissions and limitations # under the License. import re +import tarfile +from io import BytesIO from os import environ +from pathlib import Path from typing import Optional from testcontainers.core.generic import DbContainer @@ -85,7 +88,7 @@ def __init__( if self.username == "root": self.root_password = self.password if seed is not None: - self.with_volume_mapping(seed, "/docker-entrypoint-initdb.d/") + self.seed = seed def _configure(self) -> None: self.with_env("MYSQL_ROOT_PASSWORD", self.root_password) @@ -105,3 +108,12 @@ def get_connection_url(self) -> str: return super()._create_connection_url( dialect="mysql+pymysql", username=self.username, password=self.password, dbname=self.dbname, port=self.port ) + + def _transfer_seed(self) -> None: + src_path = Path(self.seed) + dest_path = "/docker-entrypoint-initdb.d/" + with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: + for filename in src_path.iterdir(): + tar.add(filename.absolute(), arcname=filename.relative_to(src_path)) + archive.seek(0) + self.get_wrapped_container().put_archive(dest_path, archive) From 8a79bd3d989d357d0e2c22a9220c3aed4b4ccdde Mon Sep 17 00:00:00 2001 From: Jb DOYON Date: Fri, 26 Apr 2024 02:29:34 +0100 Subject: [PATCH 4/4] Fix the feature Stop waiting for connection, which requires ready container. Instead copy data first thing. --- core/testcontainers/core/generic.py | 2 +- modules/mysql/testcontainers/mysql/__init__.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/testcontainers/core/generic.py b/core/testcontainers/core/generic.py index 23c24ea7..6dd635e6 100644 --- a/core/testcontainers/core/generic.py +++ b/core/testcontainers/core/generic.py @@ -70,8 +70,8 @@ def _create_connection_url( def start(self) -> "DbContainer": self._configure() super().start() - self._connect() self._transfer_seed() + self._connect() return self def _configure(self) -> None: diff --git a/modules/mysql/testcontainers/mysql/__init__.py b/modules/mysql/testcontainers/mysql/__init__.py index c7ed97b7..46efbcfb 100644 --- a/modules/mysql/testcontainers/mysql/__init__.py +++ b/modules/mysql/testcontainers/mysql/__init__.py @@ -87,8 +87,7 @@ def __init__( if self.username == "root": self.root_password = self.password - if seed is not None: - self.seed = seed + self.seed = seed def _configure(self) -> None: self.with_env("MYSQL_ROOT_PASSWORD", self.root_password) @@ -110,6 +109,8 @@ def get_connection_url(self) -> str: ) def _transfer_seed(self) -> None: + if self.seed is None: + return src_path = Path(self.seed) dest_path = "/docker-entrypoint-initdb.d/" with BytesIO() as archive, tarfile.TarFile(fileobj=archive, mode="w") as tar: