Skip to content

Commit c69b8d2

Browse files
franklieefrankliee
authored
Tests: Unify the test catalog setting (#609)
1 parent 49ac3a2 commit c69b8d2

File tree

2 files changed

+60
-116
lines changed

2 files changed

+60
-116
lines changed

‎tests/integration/test_partition_evolution.py

Lines changed: 29 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
import pytest
2020

21-
from pyiceberg.catalog import Catalog, load_catalog
21+
from pyiceberg.catalog import Catalog
2222
from pyiceberg.exceptions import NoSuchTableError
2323
from pyiceberg.partitioning import PartitionField, PartitionSpec
2424
from pyiceberg.schema import Schema
@@ -41,34 +41,6 @@
4141
)
4242

4343

44-
@pytest.fixture()
45-
def catalog_rest() -> Catalog:
46-
return load_catalog(
47-
"local",
48-
**{
49-
"type": "rest",
50-
"uri": "http://localhost:8181",
51-
"s3.endpoint": "http://localhost:9000",
52-
"s3.access-key-id": "admin",
53-
"s3.secret-access-key": "password",
54-
},
55-
)
56-
57-
58-
@pytest.fixture()
59-
def catalog_hive() -> Catalog:
60-
return load_catalog(
61-
"local",
62-
**{
63-
"type": "hive",
64-
"uri": "http://localhost:9083",
65-
"s3.endpoint": "http://localhost:9000",
66-
"s3.access-key-id": "admin",
67-
"s3.secret-access-key": "password",
68-
},
69-
)
70-
71-
7244
def _simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
7345
return _create_table_with_schema(catalog, table_schema_simple, "1")
7446

@@ -101,7 +73,7 @@ def _create_table_with_schema(catalog: Catalog, schema: Schema, format_version:
10173

10274

10375
@pytest.mark.integration
104-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
76+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
10577
def test_add_identity_partition(catalog: Catalog, table_schema_simple: Schema) -> None:
10678
simple_table = _simple_table(catalog, table_schema_simple)
10779
simple_table.update_spec().add_identity("foo").commit()
@@ -113,47 +85,47 @@ def test_add_identity_partition(catalog: Catalog, table_schema_simple: Schema) -
11385

11486

11587
@pytest.mark.integration
116-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
88+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
11789
def test_add_year(catalog: Catalog) -> None:
11890
table = _table(catalog)
11991
table.update_spec().add_field("event_ts", YearTransform(), "year_transform").commit()
12092
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, YearTransform(), "year_transform"))
12193

12294

12395
@pytest.mark.integration
124-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
96+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
12597
def test_add_month(catalog: Catalog) -> None:
12698
table = _table(catalog)
12799
table.update_spec().add_field("event_ts", MonthTransform(), "month_transform").commit()
128100
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, MonthTransform(), "month_transform"))
129101

130102

131103
@pytest.mark.integration
132-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
104+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
133105
def test_add_day(catalog: Catalog) -> None:
134106
table = _table(catalog)
135107
table.update_spec().add_field("event_ts", DayTransform(), "day_transform").commit()
136108
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, DayTransform(), "day_transform"))
137109

138110

139111
@pytest.mark.integration
140-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
112+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
141113
def test_add_hour(catalog: Catalog) -> None:
142114
table = _table(catalog)
143115
table.update_spec().add_field("event_ts", HourTransform(), "hour_transform").commit()
144116
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, HourTransform(), "hour_transform"))
145117

146118

147119
@pytest.mark.integration
148-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
120+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
149121
def test_add_bucket(catalog: Catalog, table_schema_simple: Schema) -> None:
150122
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
151123
simple_table.update_spec().add_field("foo", BucketTransform(12), "bucket_transform").commit()
152124
_validate_new_partition_fields(simple_table, 1000, 1, 1000, PartitionField(1, 1000, BucketTransform(12), "bucket_transform"))
153125

154126

155127
@pytest.mark.integration
156-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
128+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
157129
def test_add_truncate(catalog: Catalog, table_schema_simple: Schema) -> None:
158130
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
159131
simple_table.update_spec().add_field("foo", TruncateTransform(1), "truncate_transform").commit()
@@ -163,7 +135,7 @@ def test_add_truncate(catalog: Catalog, table_schema_simple: Schema) -> None:
163135

164136

165137
@pytest.mark.integration
166-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
138+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
167139
def test_multiple_adds(catalog: Catalog) -> None:
168140
table = _table(catalog)
169141
table.update_spec().add_identity("id").add_field("event_ts", HourTransform(), "hourly_partitioned").add_field(
@@ -181,7 +153,7 @@ def test_multiple_adds(catalog: Catalog) -> None:
181153

182154

183155
@pytest.mark.integration
184-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
156+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
185157
def test_add_hour_to_day(catalog: Catalog) -> None:
186158
table = _table(catalog)
187159
table.update_spec().add_field("event_ts", DayTransform(), "daily_partitioned").commit()
@@ -197,7 +169,7 @@ def test_add_hour_to_day(catalog: Catalog) -> None:
197169

198170

199171
@pytest.mark.integration
200-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
172+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
201173
def test_add_multiple_buckets(catalog: Catalog) -> None:
202174
table = _table(catalog)
203175
table.update_spec().add_field("id", BucketTransform(16)).add_field("id", BucketTransform(4)).commit()
@@ -212,7 +184,7 @@ def test_add_multiple_buckets(catalog: Catalog) -> None:
212184

213185

214186
@pytest.mark.integration
215-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
187+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
216188
def test_remove_identity(catalog: Catalog) -> None:
217189
table = _table(catalog)
218190
table.update_spec().add_identity("id").commit()
@@ -225,7 +197,7 @@ def test_remove_identity(catalog: Catalog) -> None:
225197

226198

227199
@pytest.mark.integration
228-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
200+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
229201
def test_remove_identity_v2(catalog: Catalog) -> None:
230202
table_v2 = _table_v2(catalog)
231203
table_v2.update_spec().add_identity("id").commit()
@@ -236,7 +208,7 @@ def test_remove_identity_v2(catalog: Catalog) -> None:
236208

237209

238210
@pytest.mark.integration
239-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
211+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
240212
def test_remove_bucket(catalog: Catalog) -> None:
241213
table = _table(catalog)
242214
with table.update_spec() as update:
@@ -257,7 +229,7 @@ def test_remove_bucket(catalog: Catalog) -> None:
257229

258230

259231
@pytest.mark.integration
260-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
232+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
261233
def test_remove_bucket_v2(catalog: Catalog) -> None:
262234
table_v2 = _table_v2(catalog)
263235
with table_v2.update_spec() as update:
@@ -272,7 +244,7 @@ def test_remove_bucket_v2(catalog: Catalog) -> None:
272244

273245

274246
@pytest.mark.integration
275-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
247+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
276248
def test_remove_day(catalog: Catalog) -> None:
277249
table = _table(catalog)
278250
with table.update_spec() as update:
@@ -293,7 +265,7 @@ def test_remove_day(catalog: Catalog) -> None:
293265

294266

295267
@pytest.mark.integration
296-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
268+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
297269
def test_remove_day_v2(catalog: Catalog) -> None:
298270
table_v2 = _table_v2(catalog)
299271
with table_v2.update_spec() as update:
@@ -308,7 +280,7 @@ def test_remove_day_v2(catalog: Catalog) -> None:
308280

309281

310282
@pytest.mark.integration
311-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
283+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
312284
def test_rename(catalog: Catalog) -> None:
313285
table = _table(catalog)
314286
table.update_spec().add_identity("id").commit()
@@ -319,7 +291,7 @@ def test_rename(catalog: Catalog) -> None:
319291

320292

321293
@pytest.mark.integration
322-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
294+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
323295
def test_cannot_add_and_remove(catalog: Catalog) -> None:
324296
table = _table(catalog)
325297
with pytest.raises(ValueError) as exc_info:
@@ -328,7 +300,7 @@ def test_cannot_add_and_remove(catalog: Catalog) -> None:
328300

329301

330302
@pytest.mark.integration
331-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
303+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
332304
def test_cannot_add_redundant_time_partition(catalog: Catalog) -> None:
333305
table = _table(catalog)
334306
with pytest.raises(ValueError) as exc_info:
@@ -339,7 +311,7 @@ def test_cannot_add_redundant_time_partition(catalog: Catalog) -> None:
339311

340312

341313
@pytest.mark.integration
342-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
314+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
343315
def test_cannot_delete_and_rename(catalog: Catalog) -> None:
344316
table = _table(catalog)
345317
with pytest.raises(ValueError) as exc_info:
@@ -349,7 +321,7 @@ def test_cannot_delete_and_rename(catalog: Catalog) -> None:
349321

350322

351323
@pytest.mark.integration
352-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
324+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
353325
def test_cannot_rename_and_delete(catalog: Catalog) -> None:
354326
table = _table(catalog)
355327
with pytest.raises(ValueError) as exc_info:
@@ -359,7 +331,7 @@ def test_cannot_rename_and_delete(catalog: Catalog) -> None:
359331

360332

361333
@pytest.mark.integration
362-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
334+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
363335
def test_cannot_add_same_tranform_for_same_field(catalog: Catalog) -> None:
364336
table = _table(catalog)
365337
with pytest.raises(ValueError) as exc_info:
@@ -370,7 +342,7 @@ def test_cannot_add_same_tranform_for_same_field(catalog: Catalog) -> None:
370342

371343

372344
@pytest.mark.integration
373-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
345+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
374346
def test_cannot_add_same_field_multiple_times(catalog: Catalog) -> None:
375347
table = _table(catalog)
376348
with pytest.raises(ValueError) as exc_info:
@@ -381,7 +353,7 @@ def test_cannot_add_same_field_multiple_times(catalog: Catalog) -> None:
381353

382354

383355
@pytest.mark.integration
384-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
356+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
385357
def test_cannot_add_multiple_specs_same_name(catalog: Catalog) -> None:
386358
table = _table(catalog)
387359
with pytest.raises(ValueError) as exc_info:
@@ -392,7 +364,7 @@ def test_cannot_add_multiple_specs_same_name(catalog: Catalog) -> None:
392364

393365

394366
@pytest.mark.integration
395-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
367+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
396368
def test_change_specs_and_schema_transaction(catalog: Catalog) -> None:
397369
table = _table(catalog)
398370
with table.transaction() as transaction:
@@ -425,7 +397,7 @@ def test_change_specs_and_schema_transaction(catalog: Catalog) -> None:
425397

426398

427399
@pytest.mark.integration
428-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
400+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
429401
def test_multiple_adds_and_remove_v1(catalog: Catalog) -> None:
430402
table = _table(catalog)
431403
with table.update_spec() as update:
@@ -447,7 +419,7 @@ def test_multiple_adds_and_remove_v1(catalog: Catalog) -> None:
447419

448420

449421
@pytest.mark.integration
450-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
422+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
451423
def test_multiple_adds_and_remove_v2(catalog: Catalog) -> None:
452424
table_v2 = _table_v2(catalog)
453425
with table_v2.update_spec() as update:
@@ -461,7 +433,7 @@ def test_multiple_adds_and_remove_v2(catalog: Catalog) -> None:
461433

462434

463435
@pytest.mark.integration
464-
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('catalog_hive'), pytest.lazy_fixture('catalog_rest')])
436+
@pytest.mark.parametrize('catalog', [pytest.lazy_fixture('session_catalog_hive'), pytest.lazy_fixture('session_catalog')])
465437
def test_multiple_remove_and_add_reuses_v2(catalog: Catalog) -> None:
466438
table_v2 = _table_v2(catalog)
467439
with table_v2.update_spec() as update:

0 commit comments

Comments
 (0)