From fcf379f5b06515947c90655c86ee3aba9422df89 Mon Sep 17 00:00:00 2001 From: Marko Grujic Date: Fri, 31 Jan 2025 09:54:20 +0100 Subject: [PATCH] Extract the common namespace and schema as fixtures for the shared tests --- .../tests/{main.rs => shared.rs} | 2 +- crates/integration_tests/tests/shared/mod.rs | 24 ------ .../append_data_file_test.rs | 32 ++------ .../append_partition_data_file_test.rs | 36 ++------- .../conflict_commit_test.rs | 30 +------- .../{shared => shared_tests}/datafusion.rs | 0 .../tests/shared_tests/mod.rs | 73 +++++++++++++++++++ .../read_evolved_schema.rs | 0 .../read_positional_deletes.rs | 0 .../{shared => shared_tests}/scan_all_type.rs | 17 +---- 10 files changed, 93 insertions(+), 121 deletions(-) rename crates/integration_tests/tests/{main.rs => shared.rs} (98%) delete mode 100644 crates/integration_tests/tests/shared/mod.rs rename crates/integration_tests/tests/{shared => shared_tests}/append_data_file_test.rs (83%) rename crates/integration_tests/tests/{shared => shared_tests}/append_partition_data_file_test.rs (86%) rename crates/integration_tests/tests/{shared => shared_tests}/conflict_commit_test.rs (81%) rename crates/integration_tests/tests/{shared => shared_tests}/datafusion.rs (100%) create mode 100644 crates/integration_tests/tests/shared_tests/mod.rs rename crates/integration_tests/tests/{shared => shared_tests}/read_evolved_schema.rs (100%) rename crates/integration_tests/tests/{shared => shared_tests}/read_positional_deletes.rs (100%) rename crates/integration_tests/tests/{shared => shared_tests}/scan_all_type.rs (96%) diff --git a/crates/integration_tests/tests/main.rs b/crates/integration_tests/tests/shared.rs similarity index 98% rename from crates/integration_tests/tests/main.rs rename to crates/integration_tests/tests/shared.rs index 3eabf6db8..500c63cab 100644 --- a/crates/integration_tests/tests/main.rs +++ b/crates/integration_tests/tests/shared.rs @@ -20,7 +20,7 @@ use std::sync::{Arc, OnceLock}; use ctor::dtor; use iceberg_integration_tests::{set_test_fixture, TestFixture}; -pub mod shared; +pub mod shared_tests; static DOCKER_CONTAINERS: OnceLock> = OnceLock::new(); diff --git a/crates/integration_tests/tests/shared/mod.rs b/crates/integration_tests/tests/shared/mod.rs deleted file mode 100644 index 6ea52b647..000000000 --- a/crates/integration_tests/tests/shared/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod append_data_file_test; -mod append_partition_data_file_test; -mod conflict_commit_test; -mod datafusion; -mod read_evolved_schema; -mod read_positional_deletes; -mod scan_all_type; diff --git a/crates/integration_tests/tests/shared/append_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_data_file_test.rs similarity index 83% rename from crates/integration_tests/tests/shared/append_data_file_test.rs rename to crates/integration_tests/tests/shared_tests/append_data_file_test.rs index 7bee80760..3f9f00d62 100644 --- a/crates/integration_tests/tests/shared/append_data_file_test.rs +++ b/crates/integration_tests/tests/shared_tests/append_data_file_test.rs @@ -17,12 +17,10 @@ //! Integration tests for rest catalog. -use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; use futures::TryStreamExt; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::transaction::Transaction; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; use iceberg::writer::file_writer::location_generator::{ @@ -30,43 +28,23 @@ use iceberg::writer::file_writer::location_generator::{ }; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation}; +use iceberg::{Catalog, TableCreation}; use iceberg_catalog_rest::RestCatalog; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::file::properties::WriterProperties; use crate::get_shared_containers; +use crate::shared_tests::{apple_ios_ns, test_schema}; #[tokio::test] async fn test_append_data_file() { let fixture = get_shared_containers(); let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let _ = rest_catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await; - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); + let ns = apple_ios_ns().await; + let schema = test_schema(); let table_creation = TableCreation::builder() - .name("t1".to_string()) + .name("t_append_data_file".to_string()) .schema(schema.clone()) .build(); diff --git a/crates/integration_tests/tests/shared/append_partition_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs similarity index 86% rename from crates/integration_tests/tests/shared/append_partition_data_file_test.rs rename to crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs index 3736f954c..ca4348d88 100644 --- a/crates/integration_tests/tests/shared/append_partition_data_file_test.rs +++ b/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs @@ -17,15 +17,11 @@ //! Integration test for partition data file -use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; use futures::TryStreamExt; -use iceberg::spec::{ - Literal, NestedField, PrimitiveLiteral, PrimitiveType, Schema, Struct, Transform, Type, - UnboundPartitionSpec, -}; +use iceberg::spec::{Literal, PrimitiveLiteral, Struct, Transform, UnboundPartitionSpec}; use iceberg::table::Table; use iceberg::transaction::Transaction; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; @@ -34,39 +30,19 @@ use iceberg::writer::file_writer::location_generator::{ }; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation}; +use iceberg::{Catalog, TableCreation}; use iceberg_catalog_rest::RestCatalog; use parquet::file::properties::WriterProperties; use crate::get_shared_containers; +use crate::shared_tests::{apple_ios_ns, test_schema}; #[tokio::test] async fn test_append_partition_data_file() { let fixture = get_shared_containers(); let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["iceberg", "rust"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let _ = rest_catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await; - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); + let ns = apple_ios_ns().await; + let schema = test_schema(); let unbound_partition_spec = UnboundPartitionSpec::builder() .add_partition_field(2, "id", Transform::Identity) @@ -78,7 +54,7 @@ async fn test_append_partition_data_file() { .expect("could not bind to schema"); let table_creation = TableCreation::builder() - .name("t2".to_string()) + .name("t_append_partition_data_file".to_string()) .schema(schema.clone()) .partition_spec(partition_spec) .build(); diff --git a/crates/integration_tests/tests/shared/conflict_commit_test.rs b/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs similarity index 81% rename from crates/integration_tests/tests/shared/conflict_commit_test.rs rename to crates/integration_tests/tests/shared_tests/conflict_commit_test.rs index 12e149c4c..744283abb 100644 --- a/crates/integration_tests/tests/shared/conflict_commit_test.rs +++ b/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs @@ -17,12 +17,10 @@ //! Integration tests for rest catalog. -use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; use futures::TryStreamExt; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::transaction::Transaction; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; use iceberg::writer::file_writer::location_generator::{ @@ -30,39 +28,19 @@ use iceberg::writer::file_writer::location_generator::{ }; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation}; +use iceberg::{Catalog, TableCreation}; use iceberg_catalog_rest::RestCatalog; use parquet::file::properties::WriterProperties; use crate::get_shared_containers; +use crate::shared_tests::{apple_ios_ns, test_schema}; #[tokio::test] async fn test_append_data_file_conflict() { let fixture = get_shared_containers(); let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let _ = rest_catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await; - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); + let ns = apple_ios_ns().await; + let schema = test_schema(); let table_creation = TableCreation::builder() .name("t3".to_string()) diff --git a/crates/integration_tests/tests/shared/datafusion.rs b/crates/integration_tests/tests/shared_tests/datafusion.rs similarity index 100% rename from crates/integration_tests/tests/shared/datafusion.rs rename to crates/integration_tests/tests/shared_tests/datafusion.rs diff --git a/crates/integration_tests/tests/shared_tests/mod.rs b/crates/integration_tests/tests/shared_tests/mod.rs new file mode 100644 index 000000000..77b9caea0 --- /dev/null +++ b/crates/integration_tests/tests/shared_tests/mod.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; +use iceberg::{Catalog, Namespace, NamespaceIdent}; +use iceberg_catalog_rest::RestCatalog; +use tokio::sync::OnceCell; + +use crate::get_shared_containers; + +mod append_data_file_test; +mod append_partition_data_file_test; +mod conflict_commit_test; +mod datafusion; +mod read_evolved_schema; +mod read_positional_deletes; +mod scan_all_type; + +static TEST_NAMESPACE: OnceCell> = OnceCell::const_new(); + +pub async fn apple_ios_ns() -> &'static Arc { + TEST_NAMESPACE + .get_or_init(|| async { + let fixture = get_shared_containers(); + let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + + let ns = Namespace::with_properties( + NamespaceIdent::from_strs(["apple", "ios"]).unwrap(), + HashMap::from([ + ("owner".to_string(), "ray".to_string()), + ("community".to_string(), "apache".to_string()), + ]), + ); + + rest_catalog + .create_namespace(ns.name(), ns.properties().clone()) + .await + .unwrap(); + + Arc::new(ns) + }) + .await +} + +fn test_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_identifier_field_ids(vec![2]) + .with_fields(vec![ + NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + ]) + .build() + .unwrap() +} diff --git a/crates/integration_tests/tests/shared/read_evolved_schema.rs b/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs similarity index 100% rename from crates/integration_tests/tests/shared/read_evolved_schema.rs rename to crates/integration_tests/tests/shared_tests/read_evolved_schema.rs diff --git a/crates/integration_tests/tests/shared/read_positional_deletes.rs b/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs similarity index 100% rename from crates/integration_tests/tests/shared/read_positional_deletes.rs rename to crates/integration_tests/tests/shared_tests/read_positional_deletes.rs diff --git a/crates/integration_tests/tests/shared/scan_all_type.rs b/crates/integration_tests/tests/shared_tests/scan_all_type.rs similarity index 96% rename from crates/integration_tests/tests/shared/scan_all_type.rs rename to crates/integration_tests/tests/shared_tests/scan_all_type.rs index 49c9462cf..bf8e57633 100644 --- a/crates/integration_tests/tests/shared/scan_all_type.rs +++ b/crates/integration_tests/tests/shared_tests/scan_all_type.rs @@ -40,30 +40,21 @@ use iceberg::writer::file_writer::location_generator::{ }; use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation}; +use iceberg::{Catalog, TableCreation}; use iceberg_catalog_rest::RestCatalog; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::properties::WriterProperties; use uuid::Uuid; use crate::get_shared_containers; +use crate::shared_tests::apple_ios_ns; #[tokio::test] async fn test_scan_all_type() { let fixture = get_shared_containers(); let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let ns = apple_ios_ns().await; - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let _ = rest_catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await; let schema = Schema::builder() .with_schema_id(1) .with_identifier_field_ids(vec![2]) @@ -134,7 +125,7 @@ async fn test_scan_all_type() { .unwrap(); let table_creation = TableCreation::builder() - .name("t4".to_string()) + .name("t_scan_all_types".to_string()) .schema(schema.clone()) .build();