Skip to content

Commit fd569a9

Browse files
committed
Make integration tests run concurrently
By extension make them use the same set of docker containers.
1 parent 430498d commit fd569a9

18 files changed

+125
-90
lines changed

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/catalog/glue/tests/glue_catalog_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ fn before_all() {
4444
normalize_test_name(module_path!()),
4545
format!("{}/testdata/glue_catalog", env!("CARGO_MANIFEST_DIR")),
4646
);
47-
docker_compose.run();
47+
docker_compose.up();
4848
guard.replace(docker_compose);
4949
}
5050

crates/catalog/hms/tests/hms_catalog_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ fn before_all() {
4343
normalize_test_name(module_path!()),
4444
format!("{}/testdata/hms_catalog", env!("CARGO_MANIFEST_DIR")),
4545
);
46-
docker_compose.run();
46+
docker_compose.up();
4747
guard.replace(docker_compose);
4848
}
4949

crates/catalog/rest/tests/rest_catalog_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ fn before_all() {
4141
normalize_test_name(module_path!()),
4242
format!("{}/testdata/rest_catalog", env!("CARGO_MANIFEST_DIR")),
4343
);
44-
docker_compose.run();
44+
docker_compose.up();
4545
guard.replace(docker_compose);
4646
}
4747

crates/iceberg/tests/file_io_gcs_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ mod tests {
4040
normalize_test_name(module_path!()),
4141
format!("{}/testdata/file_io_gcs", env!("CARGO_MANIFEST_DIR")),
4242
);
43-
docker_compose.run();
43+
docker_compose.up();
4444
guard.replace(docker_compose);
4545
}
4646

crates/iceberg/tests/file_io_s3_test.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ mod tests {
3838
normalize_test_name(module_path!()),
3939
format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")),
4040
);
41-
docker_compose.run();
41+
docker_compose.up();
4242
guard.replace(docker_compose);
4343
}
4444

crates/integration_tests/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ rust-version = { workspace = true }
2727
[dependencies]
2828
arrow-array = { workspace = true }
2929
arrow-schema = { workspace = true }
30+
ctor = { workspace = true }
3031
datafusion = { workspace = true }
3132
futures = { workspace = true }
3233
iceberg = { workspace = true }

crates/integration_tests/src/lib.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,31 +18,32 @@
1818
use std::collections::HashMap;
1919

2020
use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY};
21-
use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig};
21+
use iceberg_catalog_rest::RestCatalogConfig;
2222
use iceberg_test_utils::docker::DockerCompose;
2323
use iceberg_test_utils::{normalize_test_name, set_up};
2424

2525
const REST_CATALOG_PORT: u16 = 8181;
2626

2727
pub struct TestFixture {
2828
pub _docker_compose: DockerCompose,
29-
pub rest_catalog: RestCatalog,
29+
pub catalog_config: RestCatalogConfig,
3030
}
3131

32-
pub async fn set_test_fixture(func: &str) -> TestFixture {
32+
pub fn set_test_fixture(func: &str) -> TestFixture {
3333
set_up();
3434
let docker_compose = DockerCompose::new(
3535
normalize_test_name(format!("{}_{func}", module_path!())),
3636
format!("{}/testdata", env!("CARGO_MANIFEST_DIR")),
3737
);
3838

39-
// Start docker compose
40-
docker_compose.run();
39+
// Stop any containers from previous runs and start new ones
40+
docker_compose.down();
41+
docker_compose.up();
4142

4243
let rest_catalog_ip = docker_compose.get_container_ip("rest");
4344
let minio_ip = docker_compose.get_container_ip("minio");
4445

45-
let config = RestCatalogConfig::builder()
46+
let catalog_config = RestCatalogConfig::builder()
4647
.uri(format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT))
4748
.props(HashMap::from([
4849
(
@@ -54,10 +55,9 @@ pub async fn set_test_fixture(func: &str) -> TestFixture {
5455
(S3_REGION.to_string(), "us-east-1".to_string()),
5556
]))
5657
.build();
57-
let rest_catalog = RestCatalog::new(config);
5858

5959
TestFixture {
6060
_docker_compose: docker_compose,
61-
rest_catalog,
61+
catalog_config,
6262
}
6363
}
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
use std::sync::{Arc, OnceLock};
2+
3+
use ctor::dtor;
4+
use iceberg_integration_tests::{set_test_fixture, TestFixture};
5+
6+
pub mod shared;
7+
8+
static DOCKER_CONTAINERS: OnceLock<Arc<TestFixture>> = OnceLock::new();
9+
10+
pub fn get_shared_containers() -> &'static Arc<TestFixture> {
11+
DOCKER_CONTAINERS.get_or_init(|| Arc::new(set_test_fixture("shared_tests")))
12+
}
13+
14+
#[dtor]
15+
fn shutdown() {
16+
if let Some(fixture) = DOCKER_CONTAINERS.get() {
17+
fixture._docker_compose.down()
18+
}
19+
}

crates/integration_tests/tests/append_data_file_test.rs renamed to crates/integration_tests/tests/shared/append_data_file_test.rs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,16 @@ use iceberg::writer::file_writer::location_generator::{
3131
use iceberg::writer::file_writer::ParquetWriterBuilder;
3232
use iceberg::writer::{IcebergWriter, IcebergWriterBuilder};
3333
use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation};
34-
use iceberg_integration_tests::set_test_fixture;
34+
use iceberg_catalog_rest::RestCatalog;
3535
use parquet::arrow::arrow_reader::ArrowReaderOptions;
3636
use parquet::file::properties::WriterProperties;
3737

38+
use crate::get_shared_containers;
39+
3840
#[tokio::test]
3941
async fn test_append_data_file() {
40-
let fixture = set_test_fixture("test_create_table").await;
42+
let fixture = get_shared_containers();
43+
let rest_catalog = RestCatalog::new(fixture.catalog_config.clone());
4144

4245
let ns = Namespace::with_properties(
4346
NamespaceIdent::from_strs(["apple", "ios"]).unwrap(),
@@ -47,11 +50,9 @@ async fn test_append_data_file() {
4750
]),
4851
);
4952

50-
fixture
51-
.rest_catalog
53+
let _ = rest_catalog
5254
.create_namespace(ns.name(), ns.properties().clone())
53-
.await
54-
.unwrap();
55+
.await;
5556

5657
let schema = Schema::builder()
5758
.with_schema_id(1)
@@ -69,8 +70,7 @@ async fn test_append_data_file() {
6970
.schema(schema.clone())
7071
.build();
7172

72-
let table = fixture
73-
.rest_catalog
73+
let table = rest_catalog
7474
.create_table(ns.name(), table_creation)
7575
.await
7676
.unwrap();
@@ -137,7 +137,7 @@ async fn test_append_data_file() {
137137
let mut append_action = tx.fast_append(None, vec![]).unwrap();
138138
append_action.add_data_files(data_file.clone()).unwrap();
139139
let tx = append_action.apply().await.unwrap();
140-
let table = tx.commit(&fixture.rest_catalog).await.unwrap();
140+
let table = tx.commit(&rest_catalog).await.unwrap();
141141

142142
// check result
143143
let batch_stream = table
@@ -157,7 +157,7 @@ async fn test_append_data_file() {
157157
let mut append_action = tx.fast_append(None, vec![]).unwrap();
158158
append_action.add_data_files(data_file.clone()).unwrap();
159159
let tx = append_action.apply().await.unwrap();
160-
let table = tx.commit(&fixture.rest_catalog).await.unwrap();
160+
let table = tx.commit(&rest_catalog).await.unwrap();
161161

162162
// check result again
163163
let batch_stream = table

0 commit comments

Comments
 (0)