Skip to content

Commit

Permalink
cherry-pick
Browse files Browse the repository at this point in the history
  • Loading branch information
HonahX committed Apr 13, 2024
1 parent f6b81f6 commit 68ba943
Showing 1 changed file with 0 additions and 22 deletions.
22 changes: 0 additions & 22 deletions tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -789,25 +789,3 @@ def test_hive_catalog_storage_descriptor(
assert len(tbl.scan().to_arrow()) == 3
# check if spark can read the table
assert spark.sql("SELECT * FROM hive.default.test_storage_descriptor").count() == 3


@pytest.mark.integration
@pytest.mark.parametrize("format_version", [1, 2])
def test_python_writes_special_character_column_with_spark_reads(
spark: SparkSession, session_catalog: Catalog, format_version: int
) -> None:
identifier = "default.python_writes_special_character_column_with_spark_reads"
column_name_with_special_character = "letter/abc"
TEST_DATA_WITH_SPECIAL_CHARACTER_COLUMN = {
column_name_with_special_character: ['a', None, 'z'],
}
pa_schema = pa.schema([
(column_name_with_special_character, pa.string()),
])
arrow_table_with_special_character_column = pa.Table.from_pydict(TEST_DATA_WITH_SPECIAL_CHARACTER_COLUMN, schema=pa_schema)
tbl = _create_table(session_catalog, identifier, {"format-version": format_version}, schema=pa_schema)

tbl.overwrite(arrow_table_with_special_character_column)
spark_df = spark.sql(f"SELECT * FROM {identifier}").toPandas()
pyiceberg_df = tbl.scan().to_pandas()
assert spark_df.equals(pyiceberg_df)

0 comments on commit 68ba943

Please sign in to comment.