diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 089a30ba61..a32a79a905 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -46,8 +46,6 @@ PREVIOUS_METADATA_LOCATION, TABLE_TYPE, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -67,7 +65,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, update_table_metadata from pyiceberg.table.metadata import TableMetadata, new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/pyiceberg/catalog/hive.py b/pyiceberg/catalog/hive.py index 4d4370fc46..b383063e3d 100644 --- a/pyiceberg/catalog/hive.py +++ b/pyiceberg/catalog/hive.py @@ -59,8 +59,6 @@ METADATA_LOCATION, TABLE_TYPE, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -79,7 +77,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, TableProperties, update_table_metadata from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/pyiceberg/catalog/noop.py b/pyiceberg/catalog/noop.py index a8b7154621..e294390e61 100644 --- a/pyiceberg/catalog/noop.py +++ b/pyiceberg/catalog/noop.py @@ -28,10 +28,9 @@ from pyiceberg.table import ( CommitTableRequest, CommitTableResponse, - SortOrder, Table, ) -from pyiceberg.table.sorting import UNSORTED_SORT_ORDER +from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties if TYPE_CHECKING: diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index a1cc1b7b70..d504d914bd 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -64,8 +64,8 @@ CommitTableResponse, Table, TableIdentifier, - TableMetadata, ) +from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder, assign_fresh_sort_order_ids from pyiceberg.typedef import EMPTY_DICT, UTF8, IcebergBaseModel diff --git a/pyiceberg/catalog/sql.py b/pyiceberg/catalog/sql.py index 0059da6676..c48a66bb1e 100644 --- a/pyiceberg/catalog/sql.py +++ b/pyiceberg/catalog/sql.py @@ -44,8 +44,6 @@ from pyiceberg.catalog import ( METADATA_LOCATION, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -64,7 +62,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table, update_table_metadata from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties if TYPE_CHECKING: import pyarrow as pa diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index a185164cd1..79bc995198 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -54,7 +54,6 @@ BoundStartsWith, BoundTerm, BoundUnaryPredicate, - L, Not, Or, UnboundPredicate, @@ -63,7 +62,7 @@ from pyiceberg.manifest import DataFile, ManifestFile, PartitionFieldSummary from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.typedef import EMPTY_DICT, StructProtocol +from pyiceberg.typedef import EMPTY_DICT, L, StructProtocol from pyiceberg.types import ( DoubleType, FloatType, diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index dbccc768be..02f72c7ca0 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -75,8 +75,8 @@ AlwaysTrue, BooleanExpression, BoundTerm, - Literal, ) +from pyiceberg.expressions.literals import Literal from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, bind, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index d73c7bc84e..fd9192bc0a 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -45,16 +45,15 @@ from sortedcontainers import SortedList from typing_extensions import Annotated +import pyiceberg.expressions.parser as parser +import pyiceberg.expressions.visitors as visitors from pyiceberg.exceptions import CommitFailedException, ResolveError, ValidationError from pyiceberg.expressions import ( AlwaysTrue, And, BooleanExpression, EqualTo, - parser, - visitors, ) -from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator, inclusive_projection from pyiceberg.io import FileIO, load_file_io from pyiceberg.manifest import ( POSITIONAL_DELETE_SCHEMA, @@ -1293,7 +1292,9 @@ def _match_deletes_to_data_file(data_entry: ManifestEntry, positional_delete_ent relevant_entries = positional_delete_entries[positional_delete_entries.bisect_right(data_entry) :] if len(relevant_entries) > 0: - evaluator = _InclusiveMetricsEvaluator(POSITIONAL_DELETE_SCHEMA, EqualTo("file_path", data_entry.data_file.file_path)) + evaluator = visitors._InclusiveMetricsEvaluator( + POSITIONAL_DELETE_SCHEMA, EqualTo("file_path", data_entry.data_file.file_path) + ) return { positional_delete_entry.data_file for positional_delete_entry in relevant_entries @@ -1317,7 +1318,7 @@ def __init__( super().__init__(table, row_filter, selected_fields, case_sensitive, snapshot_id, options, limit) def _build_partition_projection(self, spec_id: int) -> BooleanExpression: - project = inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) + project = visitors.inclusive_projection(self.table.schema(), self.table.specs()[spec_id]) return project(self.row_filter) @cached_property @@ -1384,7 +1385,7 @@ def plan_files(self) -> Iterable[FileScanTask]: # this filter depends on the partition spec used to write the manifest file partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator) - metrics_evaluator = _InclusiveMetricsEvaluator( + metrics_evaluator = visitors._InclusiveMetricsEvaluator( self.table.schema(), self.row_filter, self.case_sensitive, self.options.get("include_empty_files") == "true" ).eval diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index d15c90fee3..d869a677d6 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -30,8 +30,6 @@ from pyiceberg.catalog import ( Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -56,7 +54,7 @@ from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import IntegerType, LongType, NestedField diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 248cc14d88..004e1abcbf 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -23,7 +23,7 @@ from requests_mock import Mocker import pyiceberg -from pyiceberg.catalog import PropertiesUpdateSummary, Table, load_catalog +from pyiceberg.catalog import PropertiesUpdateSummary, load_catalog from pyiceberg.catalog.rest import AUTH_URL, RestCatalog from pyiceberg.exceptions import ( NamespaceAlreadyExistsError, @@ -35,6 +35,7 @@ from pyiceberg.io import load_file_io from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.table import Table from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import IdentityTransform, TruncateTransform diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index f6ff78283b..421b148b05 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -24,7 +24,6 @@ from pytest_lazyfixture import lazy_fixture from sqlalchemy.exc import ArgumentError, IntegrityError -from pyiceberg.catalog import Identifier from pyiceberg.catalog.sql import SqlCatalog from pyiceberg.exceptions import ( CommitFailedException, @@ -47,6 +46,7 @@ SortOrder, ) from pyiceberg.transforms import IdentityTransform +from pyiceberg.typedef import Identifier from pyiceberg.types import IntegerType diff --git a/tests/expressions/test_parser.py b/tests/expressions/test_parser.py index 3ce2f2226c..0bccc9b80f 100644 --- a/tests/expressions/test_parser.py +++ b/tests/expressions/test_parser.py @@ -17,6 +17,7 @@ import pytest from pyparsing import ParseException +import pyiceberg.expressions.parser as parser from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, @@ -37,7 +38,6 @@ NotStartsWith, Or, StartsWith, - parser, ) diff --git a/tests/integration/test_writes.py b/tests/integration/test_writes.py index 54b647b8ed..b138b2c7e2 100644 --- a/tests/integration/test_writes.py +++ b/tests/integration/test_writes.py @@ -31,10 +31,12 @@ from pyspark.sql import SparkSession from pytest_mock.plugin import MockerFixture -from pyiceberg.catalog import Catalog, Properties, Table, load_catalog +from pyiceberg.catalog import Catalog, load_catalog from pyiceberg.catalog.sql import SqlCatalog from pyiceberg.exceptions import NamespaceAlreadyExistsError, NoSuchTableError from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.typedef import Properties from pyiceberg.types import ( BinaryType, BooleanType, diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a3dd56db7f..2acffdfdf9 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -53,8 +53,8 @@ GreaterThan, Not, Or, - literal, ) +from pyiceberg.expressions.literals import literal from pyiceberg.io import InputStream, OutputStream, load_file_io from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, diff --git a/tests/table/test_init.py b/tests/table/test_init.py index efee43b192..d2c1082f83 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -53,7 +53,6 @@ RemovePropertiesUpdate, SetPropertiesUpdate, SetSnapshotRefUpdate, - SnapshotRef, StaticTable, Table, UpdateSchema, @@ -64,6 +63,7 @@ update_table_metadata, ) from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadataUtil, TableMetadataV2 +from pyiceberg.table.refs import SnapshotRef from pyiceberg.table.snapshots import ( Operation, Snapshot, diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 19c9c267c1..e0778b956d 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -29,7 +29,6 @@ from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.serializers import FromByteStream -from pyiceberg.table import SortOrder from pyiceberg.table.metadata import ( TableMetadataUtil, TableMetadataV1, @@ -37,7 +36,7 @@ new_table_metadata, ) from pyiceberg.table.refs import SnapshotRef, SnapshotRefType -from pyiceberg.table.sorting import NullOrder, SortDirection, SortField +from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import UTF8 from pyiceberg.types import ( diff --git a/tests/test_schema.py b/tests/test_schema.py index cfee6e7f14..72ce6b0cc5 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,8 +23,8 @@ from pyiceberg import schema from pyiceberg.exceptions import ResolveError, ValidationError -from pyiceberg.expressions import Accessor from pyiceberg.schema import ( + Accessor, Schema, build_position_accessors, promote, diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 6ef11a47ea..3e789cb854 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -37,8 +37,7 @@ ) from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Snapshot -from pyiceberg.table.snapshots import Operation, Summary +from pyiceberg.table.snapshots import Operation, Snapshot, Summary from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import Record from pyiceberg.types import IntegerType, NestedField