diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 4f7b5a3292..39836deca5 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -596,7 +596,7 @@ def identifier_to_database_and_table( ) -> Tuple[str, str]: tuple_identifier = Catalog.identifier_to_tuple(identifier) if len(tuple_identifier) != 2: - raise err(f"Invalid path, hierarchical namespaces are not supported: {identifier}") + raise err(f"Table does not exist: {tuple_identifier}") return tuple_identifier[0], tuple_identifier[1] diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index d44d4996b6..bb42cd3b37 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -175,7 +175,7 @@ def create_table( database_name, table_name = self.identifier_to_database_and_table(identifier) if not self._namespace_exists(database_name): - raise NoSuchNamespaceError(f"Namespace does not exist: {database_name}") + self.create_namespace(database_name) location = self._resolve_table_location(location, database_name, table_name) metadata_location = self._get_metadata_location(location=location) @@ -263,7 +263,7 @@ def load_table(self, identifier: Union[str, Identifier]) -> Table: result = session.scalar(stmt) if result: return self._convert_orm_to_iceberg(result) - raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") + raise NoSuchTableError(f"Table does not exist: {identifier_tuple}") def drop_table(self, identifier: Union[str, Identifier]) -> None: """Drop a table. @@ -286,7 +286,7 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: ) ) if res.rowcount < 1: - raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") + raise NoSuchTableError(f"Table does not exist: {identifier_tuple}") else: try: tbl = ( @@ -301,7 +301,7 @@ def drop_table(self, identifier: Union[str, Identifier]) -> None: ) session.delete(tbl) except NoResultFound as e: - raise NoSuchTableError(f"Table does not exist: {database_name}.{table_name}") from e + raise NoSuchTableError(f"Table does not exist: {identifier_tuple}") from e session.commit() def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: Union[str, Identifier]) -> Table: @@ -501,6 +501,8 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None: ) ) session.commit() + else: + raise NoSuchNamespaceError(f"Namespace does not exist: {self.identifier_to_tuple(namespace)}") def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]: """List tables under the given namespace in the catalog. @@ -537,6 +539,10 @@ def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identi Raises: NoSuchNamespaceError: If a namespace with the given name does not exist. """ + # Hierarchical namespace is not supported. Return an empty list + if namespace: + return [] + if namespace and not self._namespace_exists(namespace): raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") @@ -573,7 +579,9 @@ def load_namespace_properties(self, namespace: Union[str, Identifier]) -> Proper IcebergNamespaceProperties.catalog_name == self.name, IcebergNamespaceProperties.namespace == database_name ) with Session(self.engine) as session: - result = session.scalars(stmt) + result = list(session.scalars(stmt)) + if len(result) == 0: + raise NoSuchNamespaceError(f"Namespace does not exist: {namespace}") return {props.property_key: props.property_value for props in result} def update_namespace_properties( @@ -592,7 +600,7 @@ def update_namespace_properties( """ database_name = self.identifier_to_database(namespace) if not self._namespace_exists(database_name): - raise NoSuchNamespaceError(f"Database {database_name} does not exists") + raise NoSuchNamespaceError(f"Namespace does not exist: {self.identifier_to_tuple(namespace)}") current_properties = self.load_namespace_properties(namespace=namespace) properties_update_summary = self._get_updated_props_and_update_summary( diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index 3c208c0ab1..b02d2ada13 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -17,19 +17,25 @@ import datetime import os import uuid -from pathlib import PosixPath +from pathlib import Path, PosixPath +from typing import Optional, Union from unittest.mock import MagicMock +import pyarrow as pa import pytest from click.testing import CliRunner +from pytest import TempPathFactory from pytest_mock import MockFixture +from pyiceberg.catalog import WAREHOUSE_LOCATION, Catalog from pyiceberg.cli.console import run from pyiceberg.io import WAREHOUSE -from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform -from pyiceberg.typedef import Properties +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import LongType, NestedField from pyiceberg.utils.config import Config from tests.catalog.test_base import InMemoryCatalog @@ -52,8 +58,25 @@ def env_vars(mocker: MockFixture) -> None: mocker.patch.dict(os.environ, MOCK_ENVIRONMENT) +@pytest.fixture(name="warehouse", scope="session") +def fixture_warehouse(tmp_path_factory: TempPathFactory) -> Path: + return tmp_path_factory.mktemp("test_sql") + + +@pytest.fixture() +def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None: + datetime_mock = MagicMock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.fromtimestamp(TEST_TIMESTAMP / 1000.0).astimezone() + monkeypatch.setattr(datetime, "datetime", datetime_mock) + + +@pytest.fixture() +def mock_uuids(mocker: MockFixture) -> None: + return mocker.patch('uuid.uuid4', return_value=TEST_TABLE_UUID) + + @pytest.fixture(name="catalog") -def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog: +def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> Catalog: in_memory_catalog = InMemoryCatalog( "test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"} ) @@ -66,11 +89,28 @@ def fixture_namespace_properties() -> Properties: return TEST_NAMESPACE_PROPERTIES.copy() -@pytest.fixture() -def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None: - datetime_mock = MagicMock(wraps=datetime.datetime) - datetime_mock.now.return_value = datetime.datetime.fromtimestamp(TEST_TIMESTAMP / 1000.0).astimezone() - monkeypatch.setattr(datetime, "datetime", datetime_mock) +def _create_table( + catalog: Catalog, + identifier: Union[str, Identifier], + schema: Union[Schema, "pa.Schema"], + location: Optional[str] = None, + partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, + sort_order: SortOrder = UNSORTED_SORT_ORDER, + properties: Properties = EMPTY_DICT, + table_uuid: Optional[uuid.UUID] = None, +) -> Table: + tbl = catalog.create_table( + identifier=identifier, + schema=schema, + location=location, + partition_spec=TEST_TABLE_PARTITION_SPEC, + sort_order=sort_order, + properties=TEST_TABLE_PROPERTIES, + ) + # override the table uuid + if table_uuid: + tbl.metadata.uuid = table_uuid + return tbl TEST_TABLE_IDENTIFIER = ("default", "my_table") @@ -89,7 +129,7 @@ def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None: MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI": "test://doesnotexist"} -def test_list_root(catalog: InMemoryCatalog) -> None: +def test_list_root(catalog: Catalog) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE) runner = CliRunner() @@ -99,8 +139,9 @@ def test_list_root(catalog: InMemoryCatalog) -> None: assert TEST_TABLE_NAMESPACE in result.output -def test_list_namespace(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_list_namespace(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -114,7 +155,7 @@ def test_list_namespace(catalog: InMemoryCatalog) -> None: assert result.output == "default.my_table\n" -def test_describe_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_describe_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -124,7 +165,7 @@ def test_describe_namespace(catalog: InMemoryCatalog, namespace_properties: Prop assert result.output == "location s3://warehouse/database/location\n" -def test_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_describe_namespace_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -135,8 +176,9 @@ def test_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: @pytest.fixture() -def test_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> None: - catalog.create_table( +def test_describe_table(catalog: Catalog, mock_datetime_now: None) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -169,7 +211,7 @@ def test_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> No ) -def test_describe_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_describe_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -178,8 +220,9 @@ def test_describe_table_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table or namespace does not exist: default.doesnotexist\n" -def test_schema(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_schema(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -197,7 +240,7 @@ def test_schema(catalog: InMemoryCatalog) -> None: ) -def test_schema_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_schema_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -206,8 +249,9 @@ def test_schema_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_spec(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_spec(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -225,7 +269,7 @@ def test_spec(catalog: InMemoryCatalog) -> None: ) -def test_spec_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_spec_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -234,8 +278,10 @@ def test_spec_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_uuid(catalog: InMemoryCatalog) -> None: - catalog.create_table( +@pytest.fixture() +def test_uuid(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -248,7 +294,7 @@ def test_uuid(catalog: InMemoryCatalog) -> None: assert result.output == """d20125c8-7284-442c-9aea-15fee620737c\n""" -def test_uuid_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_uuid_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -257,8 +303,9 @@ def test_uuid_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_location(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_location(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -266,10 +313,10 @@ def test_location(catalog: InMemoryCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["location", "default.my_table"]) assert result.exit_code == 0 - assert result.output == f"""{catalog._warehouse_location}/default/my_table\n""" + assert result.output == f"""{catalog.properties.get(WAREHOUSE_LOCATION)}/default/my_table\n""" -def test_location_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_location_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -278,8 +325,9 @@ def test_location_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_drop_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_drop_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -291,7 +339,7 @@ def test_drop_table(catalog: InMemoryCatalog) -> None: assert result.output == """Dropped table: default.my_table\n""" -def test_drop_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_drop_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -300,7 +348,7 @@ def test_drop_table_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_drop_namespace(catalog: InMemoryCatalog) -> None: +def test_drop_namespace(catalog: Catalog) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE) runner = CliRunner() @@ -309,7 +357,7 @@ def test_drop_namespace(catalog: InMemoryCatalog) -> None: assert result.output == """Dropped namespace: default\n""" -def test_drop_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_drop_namespace_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -318,24 +366,9 @@ def test_drop_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Namespace does not exist: ('doesnotexist',)\n" -def test_create_namespace(catalog: InMemoryCatalog) -> None: - runner = CliRunner() - result = runner.invoke(run, ["create", "namespace", TEST_TABLE_NAMESPACE]) - assert result.exit_code == 0 - assert result.output == """Created namespace: default\n""" - - -def test_create_namespace_already_exists(catalog: InMemoryCatalog) -> None: - catalog.create_namespace(TEST_TABLE_NAMESPACE) - - runner = CliRunner() - result = runner.invoke(run, ["create", "namespace", TEST_TABLE_NAMESPACE]) - assert result.exit_code == 1 - assert result.output == "Namespace already exists: ('default',)\n" - - -def test_rename_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_rename_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -347,17 +380,18 @@ def test_rename_table(catalog: InMemoryCatalog) -> None: assert result.output == """Renamed table from default.my_table to default.my_new_table\n""" -def test_rename_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_rename_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() result = runner.invoke(run, ["rename", "default.doesnotexist", "default.bar"]) assert result.exit_code == 1 - assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" + assert result.output == "Namespace does not exist: default\n" -def test_properties_get_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_get_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -370,8 +404,9 @@ def test_properties_get_table(catalog: InMemoryCatalog) -> None: assert result.output == "read.split.target.size 134217728\n" -def test_properties_get_table_specific_property(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_get_table_specific_property(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -384,8 +419,9 @@ def test_properties_get_table_specific_property(catalog: InMemoryCatalog) -> Non assert result.output == "134217728\n" -def test_properties_get_table_specific_property_that_doesnt_exist(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_get_table_specific_property_that_doesnt_exist(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -398,7 +434,7 @@ def test_properties_get_table_specific_property_that_doesnt_exist(catalog: InMem assert result.output == "Could not find property doesnotexist on table default.my_table\n" -def test_properties_get_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_properties_get_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -407,7 +443,7 @@ def test_properties_get_table_does_not_exist(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('doesnotexist',)\n" -def test_properties_get_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_properties_get_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -416,7 +452,7 @@ def test_properties_get_namespace(catalog: InMemoryCatalog, namespace_properties assert result.output == "location s3://warehouse/database/location\n" -def test_properties_get_namespace_specific_property(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_properties_get_namespace_specific_property(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -425,7 +461,7 @@ def test_properties_get_namespace_specific_property(catalog: InMemoryCatalog, na assert result.output == "s3://warehouse/database/location\n" -def test_properties_get_namespace_does_not_exist(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_properties_get_namespace_does_not_exist(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -434,7 +470,7 @@ def test_properties_get_namespace_does_not_exist(catalog: InMemoryCatalog, names assert result.output == "Namespace does not exist: ('doesnotexist',)\n" -def test_properties_set_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_properties_set_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -443,7 +479,7 @@ def test_properties_set_namespace(catalog: InMemoryCatalog, namespace_properties assert result.output == "Updated location on default\n" -def test_properties_set_namespace_that_doesnt_exist(catalog: InMemoryCatalog) -> None: +def test_properties_set_namespace_that_doesnt_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -452,8 +488,9 @@ def test_properties_set_namespace_that_doesnt_exist(catalog: InMemoryCatalog) -> assert result.output == "Namespace does not exist: ('doesnotexist',)\n" -def test_properties_set_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_set_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -465,7 +502,7 @@ def test_properties_set_table(catalog: InMemoryCatalog) -> None: assert "Writing is WIP" in result.output -def test_properties_set_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_properties_set_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -474,7 +511,7 @@ def test_properties_set_table_does_not_exist(catalog: InMemoryCatalog) -> None: assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_properties_remove_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_properties_remove_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -483,7 +520,7 @@ def test_properties_remove_namespace(catalog: InMemoryCatalog, namespace_propert assert result.output == "Property location removed from default\n" -def test_properties_remove_namespace_that_doesnt_exist(catalog: InMemoryCatalog) -> None: +def test_properties_remove_namespace_that_doesnt_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -492,8 +529,9 @@ def test_properties_remove_namespace_that_doesnt_exist(catalog: InMemoryCatalog) assert result.output == "Namespace does not exist: ('doesnotexist',)\n" -def test_properties_remove_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_remove_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -506,8 +544,9 @@ def test_properties_remove_table(catalog: InMemoryCatalog) -> None: assert "Writing is WIP" in result.output -def test_properties_remove_table_property_does_not_exists(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_properties_remove_table_property_does_not_exists(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -519,7 +558,7 @@ def test_properties_remove_table_property_does_not_exists(catalog: InMemoryCatal assert result.output == "Property doesnotexist does not exist on default.my_table\n" -def test_properties_remove_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_properties_remove_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -528,7 +567,7 @@ def test_properties_remove_table_does_not_exist(catalog: InMemoryCatalog) -> Non assert result.output == "Table does not exist: ('default', 'doesnotexist')\n" -def test_json_list_root(catalog: InMemoryCatalog) -> None: +def test_json_list_root(catalog: Catalog) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE) runner = CliRunner() @@ -537,8 +576,9 @@ def test_json_list_root(catalog: InMemoryCatalog) -> None: assert result.output == """["default"]\n""" -def test_json_list_namespace(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_list_namespace(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -550,7 +590,7 @@ def test_json_list_namespace(catalog: InMemoryCatalog) -> None: assert result.output == """["default.my_table"]\n""" -def test_json_describe_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_json_describe_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -559,7 +599,7 @@ def test_json_describe_namespace(catalog: InMemoryCatalog, namespace_properties: assert result.output == """{"location": "s3://warehouse/database/location"}\n""" -def test_json_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_describe_namespace_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -569,8 +609,9 @@ def test_json_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> No @pytest.fixture() -def test_json_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> None: - catalog.create_table( +def test_json_describe_table(catalog: Catalog, mock_datetime_now: None) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -586,7 +627,7 @@ def test_json_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) ) -def test_json_describe_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_describe_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -598,8 +639,9 @@ def test_json_describe_table_does_not_exists(catalog: InMemoryCatalog) -> None: ) -def test_json_schema(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_schema(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -614,7 +656,7 @@ def test_json_schema(catalog: InMemoryCatalog) -> None: ) -def test_json_schema_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_schema_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -623,8 +665,9 @@ def test_json_schema_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_spec(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_spec(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -636,7 +679,7 @@ def test_json_spec(catalog: InMemoryCatalog) -> None: assert result.output == """{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}\n""" -def test_json_spec_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_spec_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -645,8 +688,10 @@ def test_json_spec_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_uuid(catalog: InMemoryCatalog) -> None: - catalog.create_table( +@pytest.fixture() +def test_json_uuid(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -659,7 +704,7 @@ def test_json_uuid(catalog: InMemoryCatalog) -> None: assert result.output == """{"uuid": "d20125c8-7284-442c-9aea-15fee620737c"}\n""" -def test_json_uuid_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_uuid_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -668,8 +713,9 @@ def test_json_uuid_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_location(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_location(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -678,10 +724,10 @@ def test_json_location(catalog: InMemoryCatalog) -> None: runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.my_table"]) assert result.exit_code == 0 - assert result.output == f'"{catalog._warehouse_location}/default/my_table"\n' + assert result.output == f'"{catalog.properties.get(WAREHOUSE_LOCATION)}/default/my_table"\n' -def test_json_location_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_location_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -690,8 +736,9 @@ def test_json_location_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_drop_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_drop_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -703,7 +750,7 @@ def test_json_drop_table(catalog: InMemoryCatalog) -> None: assert result.output == """"Dropped table: default.my_table"\n""" -def test_json_drop_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_drop_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -712,7 +759,7 @@ def test_json_drop_table_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_drop_namespace(catalog: InMemoryCatalog) -> None: +def test_json_drop_namespace(catalog: Catalog) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE) runner = CliRunner() @@ -721,7 +768,7 @@ def test_json_drop_namespace(catalog: InMemoryCatalog) -> None: assert result.output == """"Dropped namespace: default"\n""" -def test_json_drop_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_drop_namespace_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -730,8 +777,9 @@ def test_json_drop_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: ('doesnotexist',)"}\n""" -def test_json_rename_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_rename_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -743,17 +791,18 @@ def test_json_rename_table(catalog: InMemoryCatalog) -> None: assert result.output == """"Renamed table from default.my_table to default.my_new_table"\n""" -def test_json_rename_table_does_not_exists(catalog: InMemoryCatalog) -> None: +def test_json_rename_table_does_not_exists(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() result = runner.invoke(run, ["--output=json", "rename", "default.doesnotexist", "default.bar"]) assert result.exit_code == 1 - assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" + assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: default"}\n""" -def test_json_properties_get_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_get_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -766,8 +815,9 @@ def test_json_properties_get_table(catalog: InMemoryCatalog) -> None: assert result.output == """{"read.split.target.size": "134217728"}\n""" -def test_json_properties_get_table_specific_property(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_get_table_specific_property(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -780,8 +830,9 @@ def test_json_properties_get_table_specific_property(catalog: InMemoryCatalog) - assert result.output == """"134217728"\n""" -def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -797,7 +848,7 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog: ) -def test_json_properties_get_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_get_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -806,7 +857,7 @@ def test_json_properties_get_table_does_not_exist(catalog: InMemoryCatalog) -> N assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('doesnotexist',)"}\n""" -def test_json_properties_get_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_json_properties_get_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -815,7 +866,7 @@ def test_json_properties_get_namespace(catalog: InMemoryCatalog, namespace_prope assert result.output == """{"location": "s3://warehouse/database/location"}\n""" -def test_json_properties_get_namespace_specific_property(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_json_properties_get_namespace_specific_property(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -824,7 +875,7 @@ def test_json_properties_get_namespace_specific_property(catalog: InMemoryCatalo assert result.output == """"s3://warehouse/database/location"\n""" -def test_json_properties_get_namespace_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_get_namespace_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -833,7 +884,7 @@ def test_json_properties_get_namespace_does_not_exist(catalog: InMemoryCatalog) assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: ('doesnotexist',)"}\n""" -def test_json_properties_set_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_json_properties_set_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -842,7 +893,7 @@ def test_json_properties_set_namespace(catalog: InMemoryCatalog, namespace_prope assert result.output == """"Updated location on default"\n""" -def test_json_properties_set_namespace_that_doesnt_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_set_namespace_that_doesnt_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -853,8 +904,9 @@ def test_json_properties_set_namespace_that_doesnt_exist(catalog: InMemoryCatalo assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: ('doesnotexist',)"}\n""" -def test_json_properties_set_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_set_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -869,7 +921,7 @@ def test_json_properties_set_table(catalog: InMemoryCatalog) -> None: assert "Writing is WIP" in result.output -def test_json_properties_set_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_set_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -880,7 +932,7 @@ def test_json_properties_set_table_does_not_exist(catalog: InMemoryCatalog) -> N assert result.output == """{"type": "NoSuchTableError", "message": "Table does not exist: ('default', 'doesnotexist')"}\n""" -def test_json_properties_remove_namespace(catalog: InMemoryCatalog, namespace_properties: Properties) -> None: +def test_json_properties_remove_namespace(catalog: Catalog, namespace_properties: Properties) -> None: catalog.create_namespace(TEST_TABLE_NAMESPACE, namespace_properties) runner = CliRunner() @@ -889,7 +941,7 @@ def test_json_properties_remove_namespace(catalog: InMemoryCatalog, namespace_pr assert result.output == """"Property location removed from default"\n""" -def test_json_properties_remove_namespace_that_doesnt_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_remove_namespace_that_doesnt_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner() @@ -898,8 +950,9 @@ def test_json_properties_remove_namespace_that_doesnt_exist(catalog: InMemoryCat assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: ('doesnotexist',)"}\n""" -def test_json_properties_remove_table(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_remove_table(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -912,8 +965,9 @@ def test_json_properties_remove_table(catalog: InMemoryCatalog) -> None: assert "Writing is WIP" in result.output -def test_json_properties_remove_table_property_does_not_exists(catalog: InMemoryCatalog) -> None: - catalog.create_table( +def test_json_properties_remove_table_property_does_not_exists(catalog: Catalog) -> None: + _create_table( + catalog, identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, partition_spec=TEST_TABLE_PARTITION_SPEC, @@ -929,7 +983,7 @@ def test_json_properties_remove_table_property_does_not_exists(catalog: InMemory ) -def test_json_properties_remove_table_does_not_exist(catalog: InMemoryCatalog) -> None: +def test_json_properties_remove_table_does_not_exist(catalog: Catalog) -> None: # pylint: disable=unused-argument runner = CliRunner()