diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index db88f7e824..54446049a4 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -59,7 +59,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 with: output-dir: wheelhouse config-file: "pyproject.toml" diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 724a45c52f..828dd18621 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -194,6 +194,16 @@ static_table = StaticTable.from_metadata( The static-table is considered read-only. +## Check if a table exists + +To check whether the `bids` table exists: + +```python +catalog.table_exists("docs_example.bids") +``` + +Returns `True` if the table already exists. + ## Write support With PyIceberg 0.6.0 write support is added through Arrow. Let's consider an Arrow Table: @@ -285,6 +295,23 @@ long: [[4.896029,-122.431297,6.0989,2.349014],[6.56667]] The nested lists indicate the different Arrow buffers, where the first write results into a buffer, and the second append in a separate buffer. This is expected since it will read two parquet files. +To avoid any type errors during writing, you can enforce the PyArrow table types using the Iceberg table schema: + +```python +from pyiceberg.catalog import load_catalog +import pyarrow as pa + +catalog = load_catalog("default") +table = catalog.load_table("default.cities") +schema = table.schema().as_arrow() + +df = pa.Table.from_pylist( + [{"city": "Groningen", "lat": 53.21917, "long": 6.56667}], schema=schema +) + +table.append(df) +``` + !!! example "Under development" @@ -292,6 +319,70 @@ The nested lists indicate the different Arrow buffers, where the first write res +## Inspecting tables + +To explore the table metadata, tables can be inspected. + +### Snapshots + +Inspect the snapshots of the table: + +```python +table.inspect.snapshots() +``` + +``` +pyarrow.Table +committed_at: timestamp[ms] not null +snapshot_id: int64 not null +parent_id: int64 +operation: string +manifest_list: string not null +summary: map + child 0, entries: struct not null + child 0, key: string not null + child 1, value: string +---- +committed_at: [[2024-03-15 15:01:25.682,2024-03-15 15:01:25.730,2024-03-15 15:01:25.772]] +snapshot_id: [[805611270568163028,3679426539959220963,5588071473139865870]] +parent_id: [[null,805611270568163028,3679426539959220963]] +operation: [["append","overwrite","append"]] +manifest_list: [["s3://warehouse/default/table_metadata_snapshots/metadata/snap-805611270568163028-0-43637daf-ea4b-4ceb-b096-a60c25481eb5.avro","s3://warehouse/default/table_metadata_snapshots/metadata/snap-3679426539959220963-0-8be81019-adf1-4bb6-a127-e15217bd50b3.avro","s3://warehouse/default/table_metadata_snapshots/metadata/snap-5588071473139865870-0-1382dd7e-5fbc-4c51-9776-a832d7d0984e.avro"]] +summary: [[keys:["added-files-size","added-data-files","added-records","total-data-files","total-delete-files","total-records","total-files-size","total-position-deletes","total-equality-deletes"]values:["5459","1","3","1","0","3","5459","0","0"],keys:["added-files-size","added-data-files","added-records","total-data-files","total-records",...,"total-equality-deletes","total-files-size","deleted-data-files","deleted-records","removed-files-size"]values:["5459","1","3","1","3",...,"0","5459","1","3","5459"],keys:["added-files-size","added-data-files","added-records","total-data-files","total-delete-files","total-records","total-files-size","total-position-deletes","total-equality-deletes"]values:["5459","1","3","2","0","6","10918","0","0"]]] +``` + +### Add Files + +Expert Iceberg users may choose to commit existing parquet files to the Iceberg table as data files, without rewriting them. + +``` +# Given that these parquet files have schema consistent with the Iceberg table + +file_paths = [ + "s3a://warehouse/default/existing-1.parquet", + "s3a://warehouse/default/existing-2.parquet", +] + +# They can be added to the table without rewriting them + +tbl.add_files(file_paths=file_paths) + +# A new snapshot is committed to the table with manifests pointing to the existing parquet files +``` + + + +!!! note "Name Mapping" + Because `add_files` uses existing files without writing new parquet files that are aware of the Iceberg's schema, it requires the Iceberg's table to have a [Name Mapping](https://iceberg.apache.org/spec/?h=name+mapping#name-mapping-serialization) (The Name mapping maps the field names within the parquet files to the Iceberg field IDs). Hence, `add_files` requires that there are no field IDs in the parquet file's metadata, and creates a new Name Mapping based on the table's current schema if the table doesn't already have one. + +!!! note "Partitions" + `add_files` only requires the client to read the existing parquet files' metadata footer to infer the partition value of each file. This implementation also supports adding files to Iceberg tables with partition transforms like `MonthTransform`, and `TruncateTransform` which preserve the order of the values after the transformation (Any Transform that has the `preserves_order` property set to True is supported). Please note that if the column statistics of the `PartitionField`'s source column are not present in the parquet metadata, the partition value is inferred as `None`. + +!!! warning "Maintenance Operations" + Because `add_files` commits the existing parquet files to the Iceberg Table as any other data file, destructive maintenance operations like expiring snapshots will remove them. + + + ## Schema evolution PyIceberg supports full schema evolution through the Python API. It takes care of setting the field-IDs and makes sure that only non-breaking changes are done (can be overriden). @@ -503,6 +594,20 @@ table = table.transaction().remove_properties("abc").commit_transaction() assert table.properties == {} ``` +## Snapshot properties + +Optionally, Snapshot properties can be set while writing to a table using `append` or `overwrite` API: + +```python +tbl.append(df, snapshot_properties={"abc": "def"}) + +# or + +tbl.overwrite(df, snapshot_properties={"abc": "def"}) + +assert tbl.metadata.snapshots[-1].summary["abc"] == "def" +``` + ## Query the data To query a table, a table scan is needed. A table scan accepts a filter, columns, optionally a limit and a snapshot ID: diff --git a/poetry.lock b/poetry.lock index 6cd3829a78..0a73144ea6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "adlfs" @@ -46,87 +46,87 @@ boto3 = ["boto3 (>=1.33.2,<1.34.35)"] [[package]] name = "aiohttp" -version = "3.9.2" +version = "3.9.3" description = "Async http client/server framework (asyncio)" optional = true python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:772fbe371788e61c58d6d3d904268e48a594ba866804d08c995ad71b144f94cb"}, - {file = "aiohttp-3.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:edd4f1af2253f227ae311ab3d403d0c506c9b4410c7fc8d9573dec6d9740369f"}, - {file = "aiohttp-3.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cfee9287778399fdef6f8a11c9e425e1cb13cc9920fd3a3df8f122500978292b"}, - {file = "aiohttp-3.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc158466f6a980a6095ee55174d1de5730ad7dec251be655d9a6a9dd7ea1ff9"}, - {file = "aiohttp-3.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54ec82f45d57c9a65a1ead3953b51c704f9587440e6682f689da97f3e8defa35"}, - {file = "aiohttp-3.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:abeb813a18eb387f0d835ef51f88568540ad0325807a77a6e501fed4610f864e"}, - {file = "aiohttp-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc91d07280d7d169f3a0f9179d8babd0ee05c79d4d891447629ff0d7d8089ec2"}, - {file = "aiohttp-3.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b65e861f4bebfb660f7f0f40fa3eb9f2ab9af10647d05dac824390e7af8f75b7"}, - {file = "aiohttp-3.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:04fd8ffd2be73d42bcf55fd78cde7958eeee6d4d8f73c3846b7cba491ecdb570"}, - {file = "aiohttp-3.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3d8d962b439a859b3ded9a1e111a4615357b01620a546bc601f25b0211f2da81"}, - {file = "aiohttp-3.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:8ceb658afd12b27552597cf9a65d9807d58aef45adbb58616cdd5ad4c258c39e"}, - {file = "aiohttp-3.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0e4ee4df741670560b1bc393672035418bf9063718fee05e1796bf867e995fad"}, - {file = "aiohttp-3.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2dec87a556f300d3211decf018bfd263424f0690fcca00de94a837949fbcea02"}, - {file = "aiohttp-3.9.2-cp310-cp310-win32.whl", hash = "sha256:3e1a800f988ce7c4917f34096f81585a73dbf65b5c39618b37926b1238cf9bc4"}, - {file = "aiohttp-3.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:ea510718a41b95c236c992b89fdfc3d04cc7ca60281f93aaada497c2b4e05c46"}, - {file = "aiohttp-3.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6aaa6f99256dd1b5756a50891a20f0d252bd7bdb0854c5d440edab4495c9f973"}, - {file = "aiohttp-3.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a27d8c70ad87bcfce2e97488652075a9bdd5b70093f50b10ae051dfe5e6baf37"}, - {file = "aiohttp-3.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:54287bcb74d21715ac8382e9de146d9442b5f133d9babb7e5d9e453faadd005e"}, - {file = "aiohttp-3.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb3d05569aa83011fcb346b5266e00b04180105fcacc63743fc2e4a1862a891"}, - {file = "aiohttp-3.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c8534e7d69bb8e8d134fe2be9890d1b863518582f30c9874ed7ed12e48abe3c4"}, - {file = "aiohttp-3.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd9d5b989d57b41e4ff56ab250c5ddf259f32db17159cce630fd543376bd96b"}, - {file = "aiohttp-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa6904088e6642609981f919ba775838ebf7df7fe64998b1a954fb411ffb4663"}, - {file = "aiohttp-3.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bda42eb410be91b349fb4ee3a23a30ee301c391e503996a638d05659d76ea4c2"}, - {file = "aiohttp-3.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:193cc1ccd69d819562cc7f345c815a6fc51d223b2ef22f23c1a0f67a88de9a72"}, - {file = "aiohttp-3.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b9f1cb839b621f84a5b006848e336cf1496688059d2408e617af33e3470ba204"}, - {file = "aiohttp-3.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:d22a0931848b8c7a023c695fa2057c6aaac19085f257d48baa24455e67df97ec"}, - {file = "aiohttp-3.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4112d8ba61fbd0abd5d43a9cb312214565b446d926e282a6d7da3f5a5aa71d36"}, - {file = "aiohttp-3.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c4ad4241b52bb2eb7a4d2bde060d31c2b255b8c6597dd8deac2f039168d14fd7"}, - {file = "aiohttp-3.9.2-cp311-cp311-win32.whl", hash = "sha256:ee2661a3f5b529f4fc8a8ffee9f736ae054adfb353a0d2f78218be90617194b3"}, - {file = "aiohttp-3.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:4deae2c165a5db1ed97df2868ef31ca3cc999988812e82386d22937d9d6fed52"}, - {file = "aiohttp-3.9.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:6f4cdba12539215aaecf3c310ce9d067b0081a0795dd8a8805fdb67a65c0572a"}, - {file = "aiohttp-3.9.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:84e843b33d5460a5c501c05539809ff3aee07436296ff9fbc4d327e32aa3a326"}, - {file = "aiohttp-3.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8008d0f451d66140a5aa1c17e3eedc9d56e14207568cd42072c9d6b92bf19b52"}, - {file = "aiohttp-3.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61c47ab8ef629793c086378b1df93d18438612d3ed60dca76c3422f4fbafa792"}, - {file = "aiohttp-3.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc71f748e12284312f140eaa6599a520389273174b42c345d13c7e07792f4f57"}, - {file = "aiohttp-3.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1c3a4d0ab2f75f22ec80bca62385db2e8810ee12efa8c9e92efea45c1849133"}, - {file = "aiohttp-3.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a87aa0b13bbee025faa59fa58861303c2b064b9855d4c0e45ec70182bbeba1b"}, - {file = "aiohttp-3.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2cc0d04688b9f4a7854c56c18aa7af9e5b0a87a28f934e2e596ba7e14783192"}, - {file = "aiohttp-3.9.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1956e3ac376b1711c1533266dec4efd485f821d84c13ce1217d53e42c9e65f08"}, - {file = "aiohttp-3.9.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:114da29f39eccd71b93a0fcacff178749a5c3559009b4a4498c2c173a6d74dff"}, - {file = "aiohttp-3.9.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3f17999ae3927d8a9a823a1283b201344a0627272f92d4f3e3a4efe276972fe8"}, - {file = "aiohttp-3.9.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:f31df6a32217a34ae2f813b152a6f348154f948c83213b690e59d9e84020925c"}, - {file = "aiohttp-3.9.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7a75307ffe31329928a8d47eae0692192327c599113d41b278d4c12b54e1bd11"}, - {file = "aiohttp-3.9.2-cp312-cp312-win32.whl", hash = "sha256:972b63d589ff8f305463593050a31b5ce91638918da38139b9d8deaba9e0fed7"}, - {file = "aiohttp-3.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:200dc0246f0cb5405c80d18ac905c8350179c063ea1587580e3335bfc243ba6a"}, - {file = "aiohttp-3.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:158564d0d1020e0d3fe919a81d97aadad35171e13e7b425b244ad4337fc6793a"}, - {file = "aiohttp-3.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:da1346cd0ccb395f0ed16b113ebb626fa43b7b07fd7344fce33e7a4f04a8897a"}, - {file = "aiohttp-3.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eaa9256de26ea0334ffa25f1913ae15a51e35c529a1ed9af8e6286dd44312554"}, - {file = "aiohttp-3.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1543e7fb00214fb4ccead42e6a7d86f3bb7c34751ec7c605cca7388e525fd0b4"}, - {file = "aiohttp-3.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:186e94570433a004e05f31f632726ae0f2c9dee4762a9ce915769ce9c0a23d89"}, - {file = "aiohttp-3.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d52d20832ac1560f4510d68e7ba8befbc801a2b77df12bd0cd2bcf3b049e52a4"}, - {file = "aiohttp-3.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c45e4e815ac6af3b72ca2bde9b608d2571737bb1e2d42299fc1ffdf60f6f9a1"}, - {file = "aiohttp-3.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa906b9bdfd4a7972dd0628dbbd6413d2062df5b431194486a78f0d2ae87bd55"}, - {file = "aiohttp-3.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:68bbee9e17d66f17bb0010aa15a22c6eb28583edcc8b3212e2b8e3f77f3ebe2a"}, - {file = "aiohttp-3.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4c189b64bd6d9a403a1a3f86a3ab3acbc3dc41a68f73a268a4f683f89a4dec1f"}, - {file = "aiohttp-3.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:8a7876f794523123bca6d44bfecd89c9fec9ec897a25f3dd202ee7fc5c6525b7"}, - {file = "aiohttp-3.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:d23fba734e3dd7b1d679b9473129cd52e4ec0e65a4512b488981a56420e708db"}, - {file = "aiohttp-3.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b141753be581fab842a25cb319f79536d19c2a51995d7d8b29ee290169868eab"}, - {file = "aiohttp-3.9.2-cp38-cp38-win32.whl", hash = "sha256:103daf41ff3b53ba6fa09ad410793e2e76c9d0269151812e5aba4b9dd674a7e8"}, - {file = "aiohttp-3.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:328918a6c2835861ff7afa8c6d2c70c35fdaf996205d5932351bdd952f33fa2f"}, - {file = "aiohttp-3.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5264d7327c9464786f74e4ec9342afbbb6ee70dfbb2ec9e3dfce7a54c8043aa3"}, - {file = "aiohttp-3.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07205ae0015e05c78b3288c1517afa000823a678a41594b3fdc870878d645305"}, - {file = "aiohttp-3.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0a1e638cffc3ec4d4784b8b4fd1cf28968febc4bd2718ffa25b99b96a741bd"}, - {file = "aiohttp-3.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d43302a30ba1166325974858e6ef31727a23bdd12db40e725bec0f759abce505"}, - {file = "aiohttp-3.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16a967685907003765855999af11a79b24e70b34dc710f77a38d21cd9fc4f5fe"}, - {file = "aiohttp-3.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6fa3ee92cd441d5c2d07ca88d7a9cef50f7ec975f0117cd0c62018022a184308"}, - {file = "aiohttp-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b500c5ad9c07639d48615a770f49618130e61be36608fc9bc2d9bae31732b8f"}, - {file = "aiohttp-3.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c07327b368745b1ce2393ae9e1aafed7073d9199e1dcba14e035cc646c7941bf"}, - {file = "aiohttp-3.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cc7d6502c23a0ec109687bf31909b3fb7b196faf198f8cff68c81b49eb316ea9"}, - {file = "aiohttp-3.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:07be2be7071723c3509ab5c08108d3a74f2181d4964e869f2504aaab68f8d3e8"}, - {file = "aiohttp-3.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:122468f6fee5fcbe67cb07014a08c195b3d4c41ff71e7b5160a7bcc41d585a5f"}, - {file = "aiohttp-3.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:00a9abcea793c81e7f8778ca195a1714a64f6d7436c4c0bb168ad2a212627000"}, - {file = "aiohttp-3.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a9825fdd64ecac5c670234d80bb52bdcaa4139d1f839165f548208b3779c6c6"}, - {file = "aiohttp-3.9.2-cp39-cp39-win32.whl", hash = "sha256:5422cd9a4a00f24c7244e1b15aa9b87935c85fb6a00c8ac9b2527b38627a9211"}, - {file = "aiohttp-3.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:7d579dcd5d82a86a46f725458418458fa43686f6a7b252f2966d359033ffc8ab"}, - {file = "aiohttp-3.9.2.tar.gz", hash = "sha256:b0ad0a5e86ce73f5368a164c10ada10504bf91869c05ab75d982c6048217fbf7"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc"}, + {file = "aiohttp-3.9.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29"}, + {file = "aiohttp-3.9.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf"}, + {file = "aiohttp-3.9.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52"}, + {file = "aiohttp-3.9.3-cp310-cp310-win32.whl", hash = "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b"}, + {file = "aiohttp-3.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2"}, + {file = "aiohttp-3.9.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b"}, + {file = "aiohttp-3.9.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e"}, + {file = "aiohttp-3.9.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4"}, + {file = "aiohttp-3.9.3-cp311-cp311-win32.whl", hash = "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5"}, + {file = "aiohttp-3.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869"}, + {file = "aiohttp-3.9.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53"}, + {file = "aiohttp-3.9.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6"}, + {file = "aiohttp-3.9.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"}, + {file = "aiohttp-3.9.3-cp312-cp312-win32.whl", hash = "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38"}, + {file = "aiohttp-3.9.3-cp312-cp312-win_amd64.whl", hash = "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528"}, + {file = "aiohttp-3.9.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168"}, + {file = "aiohttp-3.9.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321"}, + {file = "aiohttp-3.9.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2"}, + {file = "aiohttp-3.9.3-cp38-cp38-win32.whl", hash = "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63"}, + {file = "aiohttp-3.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1"}, + {file = "aiohttp-3.9.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58"}, + {file = "aiohttp-3.9.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a"}, + {file = "aiohttp-3.9.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d"}, + {file = "aiohttp-3.9.3-cp39-cp39-win32.whl", hash = "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051"}, + {file = "aiohttp-3.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc"}, + {file = "aiohttp-3.9.3.tar.gz", hash = "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7"}, ] [package.dependencies] @@ -214,13 +214,13 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p [[package]] name = "aws-sam-translator" -version = "1.83.0" +version = "1.85.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" optional = false python-versions = ">=3.8, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.83.0.tar.gz", hash = "sha256:46025ca8894a56eacd87eb0e4f9af5c01c567c9a734b97fbba353bffd56ba5dc"}, - {file = "aws_sam_translator-1.83.0-py3-none-any.whl", hash = "sha256:022246b4745cc9067f88ab2b051f49898606bcf0222e22b9da41063e5619c6f9"}, + {file = "aws-sam-translator-1.85.0.tar.gz", hash = "sha256:e41938affa128fb5bde5e1989b260bf539a96369bba3faf316ce66651351df39"}, + {file = "aws_sam_translator-1.85.0-py3-none-any.whl", hash = "sha256:e8c69a4db7279421ff6c3579cd4d43395fe9b6781f50416528e984be68e25481"}, ] [package.dependencies] @@ -249,13 +249,13 @@ wrapt = "*" [[package]] name = "azure-core" -version = "1.29.7" +version = "1.30.0" description = "Microsoft Azure Core Library for Python" optional = true python-versions = ">=3.7" files = [ - {file = "azure-core-1.29.7.tar.gz", hash = "sha256:2944faf1a7ff1558b1f457cabf60f279869cabaeef86b353bed8eb032c7d8c5e"}, - {file = "azure_core-1.29.7-py3-none-any.whl", hash = "sha256:95a7b41b4af102e5fcdfac9500fcc82ff86e936c7145a099b7848b9ac0501250"}, + {file = "azure-core-1.30.0.tar.gz", hash = "sha256:6f3a7883ef184722f6bd997262eddaf80cfe7e5b3e0caaaf8db1695695893d35"}, + {file = "azure_core-1.30.0-py3-none-any.whl", hash = "sha256:3dae7962aad109610e68c9a7abb31d79720e1d982ddf61363038d175a5025e89"}, ] [package.dependencies] @@ -408,13 +408,13 @@ files = [ [[package]] name = "certifi" -version = "2023.11.17" +version = "2024.2.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, - {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, + {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, + {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] [[package]] @@ -494,17 +494,17 @@ files = [ [[package]] name = "cfn-lint" -version = "0.84.0" +version = "0.85.2" description = "Checks CloudFormation templates for practices and behaviour that could potentially be improved" optional = false python-versions = ">=3.8, <=4.0, !=4.0" files = [ - {file = "cfn-lint-0.84.0.tar.gz", hash = "sha256:7a819ffa48ab23f775037ddb0d9330ba206d547439f69bc72f76f1183d8bc124"}, - {file = "cfn_lint-0.84.0-py3-none-any.whl", hash = "sha256:1bf18bca82c6666fd204a265a203bd307816e8d31fe476301b105c5370e306e5"}, + {file = "cfn-lint-0.85.2.tar.gz", hash = "sha256:f8a5cc55daeaaa747b8d776dcf62fe1b6bfb8cb46ae60950cbe627601facccd7"}, + {file = "cfn_lint-0.85.2-py3-none-any.whl", hash = "sha256:e7a0aafb9ad93dbe5db54cbefca92a94f2d173309218273ef997ecb048125d89"}, ] [package.dependencies] -aws-sam-translator = ">=1.83.0" +aws-sam-translator = ">=1.84.0" jschema-to-python = ">=1.2.3,<1.3.0" jsonpatch = "*" jsonschema = ">=3.0,<5" @@ -515,6 +515,17 @@ regex = ">=2021.7.1" sarif-om = ">=1.0.4,<1.1.0" sympy = ">=1.0.0" +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -708,43 +719,43 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "42.0.2" +version = "42.0.4" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be"}, - {file = "cryptography-42.0.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d"}, - {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4"}, - {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2"}, - {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529"}, - {file = "cryptography-42.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1"}, - {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1"}, - {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929"}, - {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9"}, - {file = "cryptography-42.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2"}, - {file = "cryptography-42.0.2-cp37-abi3-win32.whl", hash = "sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee"}, - {file = "cryptography-42.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee"}, - {file = "cryptography-42.0.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242"}, - {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a"}, - {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446"}, - {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90"}, - {file = "cryptography-42.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3"}, - {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589"}, - {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a"}, - {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea"}, - {file = "cryptography-42.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33"}, - {file = "cryptography-42.0.2-cp39-abi3-win32.whl", hash = "sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635"}, - {file = "cryptography-42.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6"}, - {file = "cryptography-42.0.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380"}, - {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6"}, - {file = "cryptography-42.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2"}, - {file = "cryptography-42.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"}, - {file = "cryptography-42.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008"}, - {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12"}, - {file = "cryptography-42.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a"}, - {file = "cryptography-42.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65"}, - {file = "cryptography-42.0.2.tar.gz", hash = "sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888"}, + {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449"}, + {file = "cryptography-42.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b"}, + {file = "cryptography-42.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824"}, + {file = "cryptography-42.0.4-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b"}, + {file = "cryptography-42.0.4-cp37-abi3-win32.whl", hash = "sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925"}, + {file = "cryptography-42.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923"}, + {file = "cryptography-42.0.4-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9"}, + {file = "cryptography-42.0.4-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929"}, + {file = "cryptography-42.0.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0"}, + {file = "cryptography-42.0.4-cp39-abi3-win32.whl", hash = "sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129"}, + {file = "cryptography-42.0.4-cp39-abi3-win_amd64.whl", hash = "sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20"}, + {file = "cryptography-42.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b"}, + {file = "cryptography-42.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660"}, + {file = "cryptography-42.0.4.tar.gz", hash = "sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb"}, ] [package.dependencies] @@ -838,6 +849,28 @@ files = [ {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +[[package]] +name = "deptry" +version = "0.14.0" +description = "A command line utility to check for unused, missing and transitive dependencies in a Python project." +optional = false +python-versions = ">=3.8, <4.0" +files = [ + {file = "deptry-0.14.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:951d40e40cca6b538d8b1992e9532c082757598c73249469d3f6f90cf3344c9f"}, + {file = "deptry-0.14.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:59e0808507ffb57a0ce9b8c40ecda7b4a235a0627f00485d8e6104bf71dc99f8"}, + {file = "deptry-0.14.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5e6b7485c3ece10eb94b2787258282275687a101587c6f88f65a6eadc65c4a4"}, + {file = "deptry-0.14.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d5a8299b9627dc32352caf99c5e8fdff0227539ccbe1535a6145649b5caa394"}, + {file = "deptry-0.14.0-cp38-abi3-win_amd64.whl", hash = "sha256:d55654025b567739a57f9b6b8467a21c65a30c21e834b2d8cb225618b320874c"}, + {file = "deptry-0.14.0.tar.gz", hash = "sha256:3415c65c0734f6dfafa1321c77c1c9fafb6d6f88d0f8441e5b2cfb5b3b41f71b"}, +] + +[package.dependencies] +chardet = ">=4.0.0" +click = ">=8.0.0,<9" +colorama = {version = ">=0.4.6", markers = "sys_platform == \"win32\""} +pathspec = ">=0.9.0" +tomli = {version = ">=2.0.1", markers = "python_version < \"3.11\""} + [[package]] name = "distlib" version = "0.3.8" @@ -1015,13 +1048,13 @@ typing = ["typing-extensions (>=4.8)"] [[package]] name = "flask" -version = "3.0.1" +version = "3.0.2" description = "A simple framework for building complex web applications." optional = false python-versions = ">=3.8" files = [ - {file = "flask-3.0.1-py3-none-any.whl", hash = "sha256:ca631a507f6dfe6c278ae20112cea3ff54ff2216390bf8880f6b035a5354af13"}, - {file = "flask-3.0.1.tar.gz", hash = "sha256:6489f51bb3666def6f314e15f19d50a1869a19ae0e8c9a3641ffe66c77d42403"}, + {file = "flask-3.0.2-py3-none-any.whl", hash = "sha256:3232e0e9c850d781933cf0207523d1ece087eb8d87b23777ae38456e2fbe7c6e"}, + {file = "flask-3.0.2.tar.gz", hash = "sha256:822c03f4b799204250a7ee84b1eddc40665395333973dfb9deebfe425fefcb7d"}, ] [package.dependencies] @@ -1233,13 +1266,13 @@ ray = ["packaging", "ray[client,data] (>=2.0.0)"] [[package]] name = "google-api-core" -version = "2.15.0" +version = "2.17.1" description = "Google API client core library" optional = true python-versions = ">=3.7" files = [ - {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, - {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, + {file = "google-api-core-2.17.1.tar.gz", hash = "sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95"}, + {file = "google_api_core-2.17.1-py3-none-any.whl", hash = "sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e"}, ] [package.dependencies] @@ -1255,13 +1288,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.26.2" +version = "2.28.0" description = "Google Authentication Library" optional = true python-versions = ">=3.7" files = [ - {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, - {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, + {file = "google-auth-2.28.0.tar.gz", hash = "sha256:3cfc1b6e4e64797584fb53fc9bd0b7afa9b7c0dba2004fa7dcc9349e58cc3195"}, + {file = "google_auth-2.28.0-py2.py3-none-any.whl", hash = "sha256:7634d29dcd1e101f5226a23cbc4a0c6cda6394253bf80e281d9c5c6797869c53"}, ] [package.dependencies] @@ -1533,13 +1566,13 @@ test = ["objgraph", "psutil"] [[package]] name = "identify" -version = "2.5.33" +version = "2.5.35" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.33-py2.py3-none-any.whl", hash = "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"}, - {file = "identify-2.5.33.tar.gz", hash = "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d"}, + {file = "identify-2.5.35-py2.py3-none-any.whl", hash = "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"}, + {file = "identify-2.5.35.tar.gz", hash = "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791"}, ] [package.extras] @@ -1717,19 +1750,18 @@ jsonpointer = ">=1.9" [[package]] name = "jsonpickle" -version = "3.0.2" +version = "3.0.3" description = "Python library for serializing any arbitrary object graph into JSON" optional = false python-versions = ">=3.7" files = [ - {file = "jsonpickle-3.0.2-py3-none-any.whl", hash = "sha256:4a8442d97ca3f77978afa58068768dba7bff2dbabe79a9647bc3cdafd4ef019f"}, - {file = "jsonpickle-3.0.2.tar.gz", hash = "sha256:e37abba4bfb3ca4a4647d28bb9f4706436f7b46c8a8333b4a718abafa8e46b37"}, + {file = "jsonpickle-3.0.3-py3-none-any.whl", hash = "sha256:e8d6dcc58f6722bea0321cd328fbda81c582461185688a535df02be0f699afb4"}, + {file = "jsonpickle-3.0.3.tar.gz", hash = "sha256:5691f44495327858ab3a95b9c440a79b41e35421be1a6e09a47b6c9b9421fd06"}, ] [package.extras] -docs = ["jaraco.packaging (>=3.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-black-multipy", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-flake8 (>=1.1.1)", "scikit-learn", "sqlalchemy"] -testing-libs = ["simplejson", "ujson"] +docs = ["furo", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "sphinx"] +testing = ["ecdsa", "feedparser", "gmpy2", "numpy", "pandas", "pymongo", "pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-ruff", "scikit-learn", "simplejson", "sqlalchemy", "ujson"] [[package]] name = "jsonpointer" @@ -1883,71 +1915,71 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] [[package]] name = "markupsafe" -version = "2.1.4" +version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" files = [ - {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de8153a7aae3835484ac168a9a9bdaa0c5eee4e0bc595503c95d53b942879c84"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e888ff76ceb39601c59e219f281466c6d7e66bd375b4ec1ce83bcdc68306796b"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b838c37ba596fcbfca71651a104a611543077156cb0a26fe0c475e1f152ee8"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac1ebf6983148b45b5fa48593950f90ed6d1d26300604f321c74a9ca1609f8e"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fbad3d346df8f9d72622ac71b69565e621ada2ce6572f37c2eae8dacd60385d"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d5291d98cd3ad9a562883468c690a2a238c4a6388ab3bd155b0c75dd55ece858"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a7cc49ef48a3c7a0005a949f3c04f8baa5409d3f663a1b36f0eba9bfe2a0396e"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b83041cda633871572f0d3c41dddd5582ad7d22f65a72eacd8d3d6d00291df26"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-win32.whl", hash = "sha256:0c26f67b3fe27302d3a412b85ef696792c4a2386293c53ba683a89562f9399b0"}, - {file = "MarkupSafe-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:a76055d5cb1c23485d7ddae533229039b850db711c554a12ea64a0fd8a0129e2"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9e9e3c4020aa2dc62d5dd6743a69e399ce3de58320522948af6140ac959ab863"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0042d6a9880b38e1dd9ff83146cc3c9c18a059b9360ceae207805567aacccc69"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d03fea4c4e9fd0ad75dc2e7e2b6757b80c152c032ea1d1de487461d8140efc"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab3a886a237f6e9c9f4f7d272067e712cdb4efa774bef494dccad08f39d8ae6"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abf5ebbec056817057bfafc0445916bb688a255a5146f900445d081db08cbabb"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e1a0d1924a5013d4f294087e00024ad25668234569289650929ab871231668e7"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e7902211afd0af05fbadcc9a312e4cf10f27b779cf1323e78d52377ae4b72bea"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c669391319973e49a7c6230c218a1e3044710bc1ce4c8e6eb71f7e6d43a2c131"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-win32.whl", hash = "sha256:31f57d64c336b8ccb1966d156932f3daa4fee74176b0fdc48ef580be774aae74"}, - {file = "MarkupSafe-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:54a7e1380dfece8847c71bf7e33da5d084e9b889c75eca19100ef98027bd9f56"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a76cd37d229fc385738bd1ce4cba2a121cf26b53864c1772694ad0ad348e509e"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:987d13fe1d23e12a66ca2073b8d2e2a75cec2ecb8eab43ff5624ba0ad42764bc"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5244324676254697fe5c181fc762284e2c5fceeb1c4e3e7f6aca2b6f107e60dc"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78bc995e004681246e85e28e068111a4c3f35f34e6c62da1471e844ee1446250"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4d176cfdfde84f732c4a53109b293d05883e952bbba68b857ae446fa3119b4f"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f9917691f410a2e0897d1ef99619fd3f7dd503647c8ff2475bf90c3cf222ad74"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f06e5a9e99b7df44640767842f414ed5d7bedaaa78cd817ce04bbd6fd86e2dd6"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:396549cea79e8ca4ba65525470d534e8a41070e6b3500ce2414921099cb73e8d"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-win32.whl", hash = "sha256:f6be2d708a9d0e9b0054856f07ac7070fbe1754be40ca8525d5adccdbda8f475"}, - {file = "MarkupSafe-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:5045e892cfdaecc5b4c01822f353cf2c8feb88a6ec1c0adef2a2e705eef0f656"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a07f40ef8f0fbc5ef1000d0c78771f4d5ca03b4953fc162749772916b298fc4"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d18b66fe626ac412d96c2ab536306c736c66cf2a31c243a45025156cc190dc8a"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:698e84142f3f884114ea8cf83e7a67ca8f4ace8454e78fe960646c6c91c63bfa"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a3b78a5af63ec10d8604180380c13dcd870aba7928c1fe04e881d5c792dc4e"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:15866d7f2dc60cfdde12ebb4e75e41be862348b4728300c36cdf405e258415ec"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:6aa5e2e7fc9bc042ae82d8b79d795b9a62bd8f15ba1e7594e3db243f158b5565"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54635102ba3cf5da26eb6f96c4b8c53af8a9c0d97b64bdcb592596a6255d8518"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-win32.whl", hash = "sha256:3583a3a3ab7958e354dc1d25be74aee6228938312ee875a22330c4dc2e41beb0"}, - {file = "MarkupSafe-2.1.4-cp37-cp37m-win_amd64.whl", hash = "sha256:d6e427c7378c7f1b2bef6a344c925b8b63623d3321c09a237b7cc0e77dd98ceb"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:bf1196dcc239e608605b716e7b166eb5faf4bc192f8a44b81e85251e62584bd2"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df98d4a9cd6a88d6a585852f56f2155c9cdb6aec78361a19f938810aa020954"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b835aba863195269ea358cecc21b400276747cc977492319fd7682b8cd2c253d"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23984d1bdae01bee794267424af55eef4dfc038dc5d1272860669b2aa025c9e3"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c98c33ffe20e9a489145d97070a435ea0679fddaabcafe19982fe9c971987d5"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9896fca4a8eb246defc8b2a7ac77ef7553b638e04fbf170bff78a40fa8a91474"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b0fe73bac2fed83839dbdbe6da84ae2a31c11cfc1c777a40dbd8ac8a6ed1560f"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c7556bafeaa0a50e2fe7dc86e0382dea349ebcad8f010d5a7dc6ba568eaaa789"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-win32.whl", hash = "sha256:fc1a75aa8f11b87910ffd98de62b29d6520b6d6e8a3de69a70ca34dea85d2a8a"}, - {file = "MarkupSafe-2.1.4-cp38-cp38-win_amd64.whl", hash = "sha256:3a66c36a3864df95e4f62f9167c734b3b1192cb0851b43d7cc08040c074c6279"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:765f036a3d00395a326df2835d8f86b637dbaf9832f90f5d196c3b8a7a5080cb"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:21e7af8091007bf4bebf4521184f4880a6acab8df0df52ef9e513d8e5db23411"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5c31fe855c77cad679b302aabc42d724ed87c043b1432d457f4976add1c2c3e"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7653fa39578957bc42e5ebc15cf4361d9e0ee4b702d7d5ec96cdac860953c5b4"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47bb5f0142b8b64ed1399b6b60f700a580335c8e1c57f2f15587bd072012decc"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:fe8512ed897d5daf089e5bd010c3dc03bb1bdae00b35588c49b98268d4a01e00"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:36d7626a8cca4d34216875aee5a1d3d654bb3dac201c1c003d182283e3205949"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b6f14a9cd50c3cb100eb94b3273131c80d102e19bb20253ac7bd7336118a673a"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-win32.whl", hash = "sha256:c8f253a84dbd2c63c19590fa86a032ef3d8cc18923b8049d91bcdeeb2581fbf6"}, - {file = "MarkupSafe-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:8b570a1537367b52396e53325769608f2a687ec9a4363647af1cded8928af959"}, - {file = "MarkupSafe-2.1.4.tar.gz", hash = "sha256:3aae9af4cac263007fd6309c64c6ab4506dd2b79382d9d19a1994f9240b8db4f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] [[package]] @@ -1962,48 +1994,97 @@ files = [ ] [[package]] -name = "mmhash3" -version = "3.0.1" -description = "Python wrapper for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +name = "mmh3" +version = "4.1.0" +description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." optional = false python-versions = "*" files = [ - {file = "mmhash3-3.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47deea30cd8d3d5cd52dc740902a4c70383bfe8248eac29d0877fe63e03c2713"}, - {file = "mmhash3-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecdaf4d1de617818bf05cd526ca558db6010beeba7ea9e19f695f2bdcac0e0a4"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4675585617584e9e1aafa3a90ac0ac9a437257c507748d97de8b21977e9d6745"}, - {file = "mmhash3-3.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebfd0c2af09b41f0fe1dd260799bda90a0fc7eba4477ccaeb3951527700cd58f"}, - {file = "mmhash3-3.0.1-cp310-cp310-win32.whl", hash = "sha256:68587dec7b8acdb7528fd511e295d8b5ccfe26022923a69867e1822f0fdb4c44"}, - {file = "mmhash3-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:54954ebe3a614f19ab4cfe24fa66ea093c493d9fac0533d002dd64c2540a0c99"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b172f3bd3220b0bf56fd7cc760fd8a9033def47103611d63fe867003079a1256"}, - {file = "mmhash3-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:de7895eafabc32f7c0c09a81a624921f536768de6e438e3de69e3e954a4d7072"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4b0914effe4ddd8d33149e3508564c17719465b0cc81691c4fa50d5e0e14f80"}, - {file = "mmhash3-3.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0575050ac691475938df1ad03d8738c5bd1eadef62093e76157ebb7f2df0946"}, - {file = "mmhash3-3.0.1-cp311-cp311-win32.whl", hash = "sha256:22f92f0f88f28b215357acd346362fa9f7c9fffb436beb42cc4b442b676dbaa3"}, - {file = "mmhash3-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:538240ab7936bf71b18304e5a7e7fd3c4c2fab103330ea99584bb4f777299a2b"}, - {file = "mmhash3-3.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ca791bfb311e36ce13998e4632262ed4b95da9d3461941e18b6690760171a045"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b41708f72c6aa2a49ada1f0b61e85c05cd8389ad31d463fd5bca29999a4d5f9c"}, - {file = "mmhash3-3.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ce9b4533ddc0a88ba045a27309714c3b127bd05e57fd252d1d5a71d4247ea7"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win32.whl", hash = "sha256:bfafeb96fdeb10db8767d06e1f07b6fdcddba4aaa0dd15058561a49f7ae45345"}, - {file = "mmhash3-3.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:97fe077b24c948709ed2afc749bf6285d407bc54ff12c63d2dc86678c38a0b8e"}, - {file = "mmhash3-3.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0cfd91ccd5fca1ba7ee925297131a15dfb94c714bfe6ba0fb3b1ca78b12bbfec"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d51b1005233141ce7394531af40a3f0fc1f274467bf8dff44dcf7987924fe58"}, - {file = "mmhash3-3.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:855c67b100e37df166acf79cdff58fa8f9f6c48be0d1e1b6e9ad0fa34a9661ef"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:bb3030df1334fd665427f8be8e8ce4f04aeab7f6010ce4f2c128f0099bdab96f"}, - {file = "mmhash3-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1545e1177294afe4912d5a5e401c7fa9b799dd109b30289e7af74d5b07e7c474"}, - {file = "mmhash3-3.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2479899e7dda834a671991a1098a691ab1c2eaa20c3e939d691ca4a19361cfe0"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9056196d5e3d3d844433a63d806a683f710ab3aaf1c910550c7746464bc43ae"}, - {file = "mmhash3-3.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d4c307af0bf70207305f70f131898be071d1b19a89f462b13487f5c25e8d4e"}, - {file = "mmhash3-3.0.1-cp38-cp38-win32.whl", hash = "sha256:5f885f65e329fd14bc38debac4a79eacf381e856965d9c65c4d1c6946ea190d0"}, - {file = "mmhash3-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:3b42d0bda5e1cd22c18b16887b0521060fb59d0aaaaf033feacbc0a2492d20fe"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3f333286bb87aa9dc6bd8e7960a55a27b011a401f24b889a50e6d219f65e7c9"}, - {file = "mmhash3-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6b7ef2eb95a18bcd02ce0d3e047adde3a025afd96c1d266a8a0d44574f44a307"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6ac8a5f511c60f341bf9cae462bb4941abb149d98464ba5f4f4548875b601c6"}, - {file = "mmhash3-3.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efef9e632e6e248e46f52d108a5ebd1f28edaf427b7fd47ebf97dbff4b2cab81"}, - {file = "mmhash3-3.0.1-cp39-cp39-win32.whl", hash = "sha256:bdac06d72e448c67afb12e758b203d875e29d4097bb279a38a5649d44b518ba7"}, - {file = "mmhash3-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:0baeaa20cac5f75ed14f28056826bd9d9c8b2354b382073f3fd5190708992a0d"}, - {file = "mmhash3-3.0.1.tar.gz", hash = "sha256:a00d68f4a1cc434b9501513c8a29e18ed1ddad383677d72b41d71d0d862348af"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"}, + {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"}, + {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"}, + {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"}, + {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"}, + {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"}, + {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"}, + {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"}, + {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"}, + {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"}, + {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"}, + {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"}, + {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"}, + {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"}, + {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"}, + {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"}, ] +[package.extras] +test = ["mypy (>=1.0)", "pytest (>=7.0.0)"] + [[package]] name = "moto" version = "5.0.2" @@ -2182,85 +2263,101 @@ files = [ [[package]] name = "multidict" -version = "6.0.4" +version = "6.0.5" description = "multidict implementation" optional = true python-versions = ">=3.7" files = [ - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, - {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, - {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, - {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, - {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, - {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, - {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, - {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, - {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, - {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, - {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, - {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, - {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, - {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, - {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, - {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, - {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, - {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, - {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, - {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, - {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, - {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, - {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, - {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, - {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, - {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, - {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, + {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, + {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, + {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, + {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, + {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, + {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, + {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, + {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, + {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, + {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, + {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, + {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, + {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, + {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, + {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, + {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, + {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, + {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, + {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, + {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, + {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, + {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, + {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, + {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, + {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, + {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, + {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, + {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, + {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, + {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, + {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, + {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] [[package]] @@ -2346,6 +2443,51 @@ files = [ {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = true +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + [[package]] name = "oauthlib" version = "3.2.2" @@ -2487,6 +2629,17 @@ files = [ {file = "pathable-0.4.3.tar.gz", hash = "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab"}, ] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "pbr" version = "6.0.0" @@ -2511,28 +2664,28 @@ files = [ [[package]] name = "platformdirs" -version = "4.1.0" +version = "4.2.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, - {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, + {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, + {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, ] [package.extras] -docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] [[package]] name = "pluggy" -version = "1.3.0" +version = "1.4.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, - {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, ] [package.extras] @@ -2578,22 +2731,22 @@ virtualenv = ">=20.10.0" [[package]] name = "protobuf" -version = "4.25.2" +version = "4.25.3" description = "" optional = true python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, - {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, - {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, - {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, - {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, - {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, - {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, - {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, - {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, + {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, + {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, + {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, + {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, + {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, + {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, + {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, + {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, + {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, ] [[package]] @@ -3153,13 +3306,13 @@ files = [ [[package]] name = "pytz" -version = "2023.3.post1" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = true python-versions = "*" files = [ - {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, - {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -3302,13 +3455,13 @@ tune = ["fsspec", "pandas", "pyarrow (>=6.0.1)", "requests", "tensorboardX (>=1. [[package]] name = "referencing" -version = "0.32.1" +version = "0.33.0" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" files = [ - {file = "referencing-0.32.1-py3-none-any.whl", hash = "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"}, - {file = "referencing-0.32.1.tar.gz", hash = "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3"}, + {file = "referencing-0.33.0-py3-none-any.whl", hash = "sha256:39240f2ecc770258f28b642dd47fd74bc8b02484de54e1882b74b35ebd779bd5"}, + {file = "referencing-0.33.0.tar.gz", hash = "sha256:c775fedf74bc0f9189c2a3be1c12fd03e8c23f4d371dce795df44e06c5b412f7"}, ] [package.dependencies] @@ -3477,13 +3630,13 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] [[package]] name = "responses" -version = "0.24.1" +version = "0.25.0" description = "A utility library for mocking out the `requests` Python library." optional = false python-versions = ">=3.8" files = [ - {file = "responses-0.24.1-py3-none-any.whl", hash = "sha256:a2b43f4c08bfb9c9bd242568328c65a34b318741d3fab884ac843c5ceeb543f9"}, - {file = "responses-0.24.1.tar.gz", hash = "sha256:b127c6ca3f8df0eb9cc82fd93109a3007a86acb24871834c47b77765152ecf8c"}, + {file = "responses-0.25.0-py3-none-any.whl", hash = "sha256:2f0b9c2b6437db4b528619a77e5d565e4ec2a9532162ac1a131a83529db7be1a"}, + {file = "responses-0.25.0.tar.gz", hash = "sha256:01ae6a02b4f34e39bffceb0fc6786b67a25eae919c6368d05eabc8d9576c2a66"}, ] [package.dependencies] @@ -3529,110 +3682,110 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rpds-py" -version = "0.17.1" +version = "0.18.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" files = [ - {file = "rpds_py-0.17.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d"}, - {file = "rpds_py-0.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9"}, - {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453"}, - {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc"}, - {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394"}, - {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59"}, - {file = "rpds_py-0.17.1-cp310-none-win32.whl", hash = "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d"}, - {file = "rpds_py-0.17.1-cp310-none-win_amd64.whl", hash = "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6"}, - {file = "rpds_py-0.17.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b"}, - {file = "rpds_py-0.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8"}, - {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a"}, - {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383"}, - {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd"}, - {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea"}, - {file = "rpds_py-0.17.1-cp311-none-win32.whl", hash = "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518"}, - {file = "rpds_py-0.17.1-cp311-none-win_amd64.whl", hash = "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf"}, - {file = "rpds_py-0.17.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf"}, - {file = "rpds_py-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9"}, - {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140"}, - {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2"}, - {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253"}, - {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23"}, - {file = "rpds_py-0.17.1-cp312-none-win32.whl", hash = "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1"}, - {file = "rpds_py-0.17.1-cp312-none-win_amd64.whl", hash = "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3"}, - {file = "rpds_py-0.17.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d"}, - {file = "rpds_py-0.17.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae"}, - {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4"}, - {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896"}, - {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde"}, - {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6"}, - {file = "rpds_py-0.17.1-cp38-none-win32.whl", hash = "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a"}, - {file = "rpds_py-0.17.1-cp38-none-win_amd64.whl", hash = "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb"}, - {file = "rpds_py-0.17.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a"}, - {file = "rpds_py-0.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256"}, - {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74"}, - {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4"}, - {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772"}, - {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b"}, - {file = "rpds_py-0.17.1-cp39-none-win32.whl", hash = "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f"}, - {file = "rpds_py-0.17.1-cp39-none-win_amd64.whl", hash = "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6"}, - {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb"}, - {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296"}, - {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68"}, - {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, + {file = "rpds_py-0.18.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5b4e7d8d6c9b2e8ee2d55c90b59c707ca59bc30058269b3db7b1f8df5763557e"}, + {file = "rpds_py-0.18.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c463ed05f9dfb9baebef68048aed8dcdc94411e4bf3d33a39ba97e271624f8f7"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01e36a39af54a30f28b73096dd39b6802eddd04c90dbe161c1b8dbe22353189f"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d62dec4976954a23d7f91f2f4530852b0c7608116c257833922a896101336c51"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd18772815d5f008fa03d2b9a681ae38d5ae9f0e599f7dda233c439fcaa00d40"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:923d39efa3cfb7279a0327e337a7958bff00cc447fd07a25cddb0a1cc9a6d2da"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39514da80f971362f9267c600b6d459bfbbc549cffc2cef8e47474fddc9b45b1"}, + {file = "rpds_py-0.18.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a34d557a42aa28bd5c48a023c570219ba2593bcbbb8dc1b98d8cf5d529ab1434"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:93df1de2f7f7239dc9cc5a4a12408ee1598725036bd2dedadc14d94525192fc3"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:34b18ba135c687f4dac449aa5157d36e2cbb7c03cbea4ddbd88604e076aa836e"}, + {file = "rpds_py-0.18.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0b5dcf9193625afd8ecc92312d6ed78781c46ecbf39af9ad4681fc9f464af88"}, + {file = "rpds_py-0.18.0-cp310-none-win32.whl", hash = "sha256:c4325ff0442a12113a6379af66978c3fe562f846763287ef66bdc1d57925d337"}, + {file = "rpds_py-0.18.0-cp310-none-win_amd64.whl", hash = "sha256:7223a2a5fe0d217e60a60cdae28d6949140dde9c3bcc714063c5b463065e3d66"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3a96e0c6a41dcdba3a0a581bbf6c44bb863f27c541547fb4b9711fd8cf0ffad4"}, + {file = "rpds_py-0.18.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30f43887bbae0d49113cbaab729a112251a940e9b274536613097ab8b4899cf6"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcb25daa9219b4cf3a0ab24b0eb9a5cc8949ed4dc72acb8fa16b7e1681aa3c58"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d68c93e381010662ab873fea609bf6c0f428b6d0bb00f2c6939782e0818d37bf"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b34b7aa8b261c1dbf7720b5d6f01f38243e9b9daf7e6b8bc1fd4657000062f2c"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e6d75ab12b0bbab7215e5d40f1e5b738aa539598db27ef83b2ec46747df90e1"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b8612cd233543a3781bc659c731b9d607de65890085098986dfd573fc2befe5"}, + {file = "rpds_py-0.18.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aec493917dd45e3c69d00a8874e7cbed844efd935595ef78a0f25f14312e33c6"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:661d25cbffaf8cc42e971dd570d87cb29a665f49f4abe1f9e76be9a5182c4688"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1df3659d26f539ac74fb3b0c481cdf9d725386e3552c6fa2974f4d33d78e544b"}, + {file = "rpds_py-0.18.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a1ce3ba137ed54f83e56fb983a5859a27d43a40188ba798993812fed73c70836"}, + {file = "rpds_py-0.18.0-cp311-none-win32.whl", hash = "sha256:69e64831e22a6b377772e7fb337533c365085b31619005802a79242fee620bc1"}, + {file = "rpds_py-0.18.0-cp311-none-win_amd64.whl", hash = "sha256:998e33ad22dc7ec7e030b3df701c43630b5bc0d8fbc2267653577e3fec279afa"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7f2facbd386dd60cbbf1a794181e6aa0bd429bd78bfdf775436020172e2a23f0"}, + {file = "rpds_py-0.18.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1d9a5be316c15ffb2b3c405c4ff14448c36b4435be062a7f578ccd8b01f0c4d8"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd5bf1af8efe569654bbef5a3e0a56eca45f87cfcffab31dd8dde70da5982475"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5417558f6887e9b6b65b4527232553c139b57ec42c64570569b155262ac0754f"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:56a737287efecafc16f6d067c2ea0117abadcd078d58721f967952db329a3e5c"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8f03bccbd8586e9dd37219bce4d4e0d3ab492e6b3b533e973fa08a112cb2ffc9"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4457a94da0d5c53dc4b3e4de1158bdab077db23c53232f37a3cb7afdb053a4e3"}, + {file = "rpds_py-0.18.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0ab39c1ba9023914297dd88ec3b3b3c3f33671baeb6acf82ad7ce883f6e8e157"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9d54553c1136b50fd12cc17e5b11ad07374c316df307e4cfd6441bea5fb68496"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0af039631b6de0397ab2ba16eaf2872e9f8fca391b44d3d8cac317860a700a3f"}, + {file = "rpds_py-0.18.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:84ffab12db93b5f6bad84c712c92060a2d321b35c3c9960b43d08d0f639d60d7"}, + {file = "rpds_py-0.18.0-cp312-none-win32.whl", hash = "sha256:685537e07897f173abcf67258bee3c05c374fa6fff89d4c7e42fb391b0605e98"}, + {file = "rpds_py-0.18.0-cp312-none-win_amd64.whl", hash = "sha256:e003b002ec72c8d5a3e3da2989c7d6065b47d9eaa70cd8808b5384fbb970f4ec"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:08f9ad53c3f31dfb4baa00da22f1e862900f45908383c062c27628754af2e88e"}, + {file = "rpds_py-0.18.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0013fe6b46aa496a6749c77e00a3eb07952832ad6166bd481c74bda0dcb6d58"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e32a92116d4f2a80b629778280103d2a510a5b3f6314ceccd6e38006b5e92dcb"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e541ec6f2ec456934fd279a3120f856cd0aedd209fc3852eca563f81738f6861"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bed88b9a458e354014d662d47e7a5baafd7ff81c780fd91584a10d6ec842cb73"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2644e47de560eb7bd55c20fc59f6daa04682655c58d08185a9b95c1970fa1e07"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e8916ae4c720529e18afa0b879473049e95949bf97042e938530e072fde061d"}, + {file = "rpds_py-0.18.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:465a3eb5659338cf2a9243e50ad9b2296fa15061736d6e26240e713522b6235c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ea7d4a99f3b38c37eac212dbd6ec42b7a5ec51e2c74b5d3223e43c811609e65f"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:67071a6171e92b6da534b8ae326505f7c18022c6f19072a81dcf40db2638767c"}, + {file = "rpds_py-0.18.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:41ef53e7c58aa4ef281da975f62c258950f54b76ec8e45941e93a3d1d8580594"}, + {file = "rpds_py-0.18.0-cp38-none-win32.whl", hash = "sha256:fdea4952db2793c4ad0bdccd27c1d8fdd1423a92f04598bc39425bcc2b8ee46e"}, + {file = "rpds_py-0.18.0-cp38-none-win_amd64.whl", hash = "sha256:7cd863afe7336c62ec78d7d1349a2f34c007a3cc6c2369d667c65aeec412a5b1"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5307def11a35f5ae4581a0b658b0af8178c65c530e94893345bebf41cc139d33"}, + {file = "rpds_py-0.18.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77f195baa60a54ef9d2de16fbbfd3ff8b04edc0c0140a761b56c267ac11aa467"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39f5441553f1c2aed4de4377178ad8ff8f9d733723d6c66d983d75341de265ab"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a00312dea9310d4cb7dbd7787e722d2e86a95c2db92fbd7d0155f97127bcb40"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f2fc11e8fe034ee3c34d316d0ad8808f45bc3b9ce5857ff29d513f3ff2923a1"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:586f8204935b9ec884500498ccc91aa869fc652c40c093bd9e1471fbcc25c022"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddc2f4dfd396c7bfa18e6ce371cba60e4cf9d2e5cdb71376aa2da264605b60b9"}, + {file = "rpds_py-0.18.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ddcba87675b6d509139d1b521e0c8250e967e63b5909a7e8f8944d0f90ff36f"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7bd339195d84439cbe5771546fe8a4e8a7a045417d8f9de9a368c434e42a721e"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:d7c36232a90d4755b720fbd76739d8891732b18cf240a9c645d75f00639a9024"}, + {file = "rpds_py-0.18.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6b0817e34942b2ca527b0e9298373e7cc75f429e8da2055607f4931fded23e20"}, + {file = "rpds_py-0.18.0-cp39-none-win32.whl", hash = "sha256:99f70b740dc04d09e6b2699b675874367885217a2e9f782bdf5395632ac663b7"}, + {file = "rpds_py-0.18.0-cp39-none-win_amd64.whl", hash = "sha256:6ef687afab047554a2d366e112dd187b62d261d49eb79b77e386f94644363294"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ad36cfb355e24f1bd37cac88c112cd7730873f20fb0bdaf8ba59eedf8216079f"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:36b3ee798c58ace201289024b52788161e1ea133e4ac93fba7d49da5fec0ef9e"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8a2f084546cc59ea99fda8e070be2fd140c3092dc11524a71aa8f0f3d5a55ca"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e4461d0f003a0aa9be2bdd1b798a041f177189c1a0f7619fe8c95ad08d9a45d7"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8db715ebe3bb7d86d77ac1826f7d67ec11a70dbd2376b7cc214199360517b641"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793968759cd0d96cac1e367afd70c235867831983f876a53389ad869b043c948"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e6a3af5a75363d2c9a48b07cb27c4ea542938b1a2e93b15a503cdfa8490795"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ef0befbb5d79cf32d0266f5cff01545602344eda89480e1dd88aca964260b18"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1d4acf42190d449d5e89654d5c1ed3a4f17925eec71f05e2a41414689cda02d1"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:a5f446dd5055667aabaee78487f2b5ab72e244f9bc0b2ffebfeec79051679984"}, + {file = "rpds_py-0.18.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9dbbeb27f4e70bfd9eec1be5477517365afe05a9b2c441a0b21929ee61048124"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:22806714311a69fd0af9b35b7be97c18a0fc2826e6827dbb3a8c94eac6cf7eeb"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b34ae4636dfc4e76a438ab826a0d1eed2589ca7d9a1b2d5bb546978ac6485461"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c8370641f1a7f0e0669ddccca22f1da893cef7628396431eb445d46d893e5cd"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c8362467a0fdeccd47935f22c256bec5e6abe543bf0d66e3d3d57a8fb5731863"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11a8c85ef4a07a7638180bf04fe189d12757c696eb41f310d2426895356dcf05"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b316144e85316da2723f9d8dc75bada12fa58489a527091fa1d5a612643d1a0e"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf1ea2e34868f6fbf070e1af291c8180480310173de0b0c43fc38a02929fc0e3"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e546e768d08ad55b20b11dbb78a745151acbd938f8f00d0cfbabe8b0199b9880"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4901165d170a5fde6f589acb90a6b33629ad1ec976d4529e769c6f3d885e3e80"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:618a3d6cae6ef8ec88bb76dd80b83cfe415ad4f1d942ca2a903bf6b6ff97a2da"}, + {file = "rpds_py-0.18.0-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ed4eb745efbff0a8e9587d22a84be94a5eb7d2d99c02dacf7bd0911713ed14dd"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6c81e5f372cd0dc5dc4809553d34f832f60a46034a5f187756d9b90586c2c307"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:43fbac5f22e25bee1d482c97474f930a353542855f05c1161fd804c9dc74a09d"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d7faa6f14017c0b1e69f5e2c357b998731ea75a442ab3841c0dbbbfe902d2c4"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:08231ac30a842bd04daabc4d71fddd7e6d26189406d5a69535638e4dcb88fe76"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:044a3e61a7c2dafacae99d1e722cc2d4c05280790ec5a05031b3876809d89a5c"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f26b5bd1079acdb0c7a5645e350fe54d16b17bfc5e71f371c449383d3342e17"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:482103aed1dfe2f3b71a58eff35ba105289b8d862551ea576bd15479aba01f66"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1374f4129f9bcca53a1bba0bb86bf78325a0374577cf7e9e4cd046b1e6f20e24"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:635dc434ff724b178cb192c70016cc0ad25a275228f749ee0daf0eddbc8183b1"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:bc362ee4e314870a70f4ae88772d72d877246537d9f8cb8f7eacf10884862432"}, + {file = "rpds_py-0.18.0-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:4832d7d380477521a8c1644bbab6588dfedea5e30a7d967b5fb75977c45fd77f"}, + {file = "rpds_py-0.18.0.tar.gz", hash = "sha256:42821446ee7a76f5d9f71f9e33a4fb2ffd724bb3e7f93386150b61a43115788d"}, ] [[package]] @@ -3703,18 +3856,18 @@ pbr = "*" [[package]] name = "setuptools" -version = "69.0.3" +version = "69.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, - {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -3899,13 +4052,13 @@ files = [ [[package]] name = "tqdm" -version = "4.66.1" +version = "4.66.2" description = "Fast, Extensible Progress Meter" optional = true python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, - {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, + {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"}, + {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"}, ] [package.dependencies] @@ -3930,13 +4083,13 @@ files = [ [[package]] name = "tzdata" -version = "2023.4" +version = "2024.1" description = "Provider of IANA time zone data" optional = true python-versions = ">=2" files = [ - {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, - {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [[package]] @@ -4298,4 +4451,4 @@ zstandard = ["zstandard"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "5553acdf7ad32ec9dbc74523b9f7ff241907b0d8129a40e3052750c3182e7539" +content-hash = "58712a40e7860a50fdae2d88a61b96ce68444fd2b222d7aece84f5b4a3a21681" diff --git a/pyiceberg/avro/file.py b/pyiceberg/avro/file.py index 2f21e165b4..d0da7651b7 100644 --- a/pyiceberg/avro/file.py +++ b/pyiceberg/avro/file.py @@ -35,7 +35,8 @@ TypeVar, ) -from pyiceberg.avro.codecs import KNOWN_CODECS, Codec +from pyiceberg.avro.codecs import KNOWN_CODECS +from pyiceberg.avro.codecs.codec import Codec from pyiceberg.avro.decoder import BinaryDecoder, new_decoder from pyiceberg.avro.encoder import BinaryEncoder from pyiceberg.avro.reader import Reader diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index db83658f1f..f2b46fcde7 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -46,8 +46,8 @@ CommitTableRequest, CommitTableResponse, Table, - TableMetadata, ) +from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.typedef import ( EMPTY_DICT, @@ -646,6 +646,13 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None: delete_files(io, prev_metadata_files, PREVIOUS_METADATA) delete_files(io, {table.metadata_location}, METADATA) + def table_exists(self, identifier: Union[str, Identifier]) -> bool: + try: + self.load_table(identifier) + return True + except NoSuchTableError: + return False + @staticmethod def _write_metadata(metadata: TableMetadata, io: FileIO, metadata_path: str) -> None: ToOutputFile.table_metadata(metadata, io.new_output(metadata_path)) diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index b7b0f3ddb1..266dd6353d 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -34,8 +34,6 @@ PREVIOUS_METADATA_LOCATION, TABLE_TYPE, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -56,7 +54,7 @@ from pyiceberg.table import CommitTableRequest, CommitTableResponse, Table from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder -from pyiceberg.typedef import EMPTY_DICT +from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties if TYPE_CHECKING: import pyarrow as pa diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index a5f33f02dd..9f0d054493 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -38,8 +38,6 @@ URI, WAREHOUSE_LOCATION, Catalog, - Identifier, - Properties, PropertiesUpdateSummary, ) from pyiceberg.exceptions import ( @@ -68,7 +66,7 @@ ) from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder, assign_fresh_sort_order_ids -from pyiceberg.typedef import EMPTY_DICT, UTF8, IcebergBaseModel +from pyiceberg.typedef import EMPTY_DICT, UTF8, IcebergBaseModel, Identifier, Properties from pyiceberg.types import transform_dict_value_to_str if TYPE_CHECKING: @@ -719,3 +717,11 @@ def update_namespace_properties( updated=parsed_response.updated, missing=parsed_response.missing, ) + + @retry(**_RETRY_ARGS) + def table_exists(self, identifier: Union[str, Identifier]) -> bool: + identifier_tuple = self.identifier_to_tuple_without_catalog(identifier) + response = self._session.head( + self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier_tuple)) + ) + return response.status_code == 200 diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 18cdab1556..56b544c99f 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -31,7 +31,8 @@ from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.table import Table, TableMetadata +from pyiceberg.table import Table +from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.refs import SnapshotRefType from pyiceberg.typedef import IcebergBaseModel, Identifier, Properties @@ -157,7 +158,7 @@ def describe_properties(self, properties: Properties) -> None: Console().print(output_table) def text(self, response: str) -> None: - Console().print(response) + Console(soft_wrap=True).print(response) def schema(self, schema: Schema) -> None: output_table = self._table diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index bf76be36e4..d9f66ae24a 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -609,6 +609,19 @@ def to(self, type_var: IcebergType) -> Literal: # type: ignore def _(self, _: UUIDType) -> Literal[bytes]: return self + @to.register(FixedType) + def _(self, type_var: FixedType) -> Literal[bytes]: + if len(type_var) == UUID_BYTES_LENGTH: + return FixedLiteral(self.value) + else: + raise TypeError( + f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}" + ) + + @to.register(BinaryType) + def _(self, _: BinaryType) -> Literal[bytes]: + return BinaryLiteral(self.value) + class FixedLiteral(Literal[bytes]): def __init__(self, value: bytes) -> None: diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 79bc995198..26698921b5 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -67,6 +67,7 @@ DoubleType, FloatType, IcebergType, + NestedField, PrimitiveType, StructType, TimestampType, @@ -534,7 +535,9 @@ def visit_or(self, left_result: bool, right_result: bool) -> bool: ROWS_MIGHT_MATCH = True +ROWS_MUST_MATCH = True ROWS_CANNOT_MATCH = False +ROWS_MIGHT_NOT_MATCH = False IN_PREDICATE_LIMIT = 200 @@ -1089,16 +1092,52 @@ def expression_to_plain_format( return [visit(expression, visitor) for expression in expressions] -class _InclusiveMetricsEvaluator(BoundBooleanExpressionVisitor[bool]): - struct: StructType - expr: BooleanExpression - +class _MetricsEvaluator(BoundBooleanExpressionVisitor[bool], ABC): value_counts: Dict[int, int] null_counts: Dict[int, int] nan_counts: Dict[int, int] lower_bounds: Dict[int, bytes] upper_bounds: Dict[int, bytes] + def visit_true(self) -> bool: + # all rows match + return ROWS_MIGHT_MATCH + + def visit_false(self) -> bool: + # all rows fail + return ROWS_CANNOT_MATCH + + def visit_not(self, child_result: bool) -> bool: + raise ValueError(f"NOT should be rewritten: {child_result}") + + def visit_and(self, left_result: bool, right_result: bool) -> bool: + return left_result and right_result + + def visit_or(self, left_result: bool, right_result: bool) -> bool: + return left_result or right_result + + def _contains_nulls_only(self, field_id: int) -> bool: + if (value_count := self.value_counts.get(field_id)) and (null_count := self.null_counts.get(field_id)): + return value_count == null_count + return False + + def _contains_nans_only(self, field_id: int) -> bool: + if (nan_count := self.nan_counts.get(field_id)) and (value_count := self.value_counts.get(field_id)): + return nan_count == value_count + return False + + def _is_nan(self, val: Any) -> bool: + try: + return math.isnan(val) + except TypeError: + # In the case of None or other non-numeric types + return False + + +class _InclusiveMetricsEvaluator(_MetricsEvaluator): + struct: StructType + expr: BooleanExpression + def __init__( self, schema: Schema, expr: BooleanExpression, case_sensitive: bool = True, include_empty_files: bool = False ) -> None: @@ -1128,40 +1167,11 @@ def eval(self, file: DataFile) -> bool: def _may_contain_null(self, field_id: int) -> bool: return self.null_counts is None or (field_id in self.null_counts and self.null_counts.get(field_id) is not None) - def _contains_nulls_only(self, field_id: int) -> bool: - if (value_count := self.value_counts.get(field_id)) and (null_count := self.null_counts.get(field_id)): - return value_count == null_count - return False - def _contains_nans_only(self, field_id: int) -> bool: if (nan_count := self.nan_counts.get(field_id)) and (value_count := self.value_counts.get(field_id)): return nan_count == value_count return False - def _is_nan(self, val: Any) -> bool: - try: - return math.isnan(val) - except TypeError: - # In the case of None or other non-numeric types - return False - - def visit_true(self) -> bool: - # all rows match - return ROWS_MIGHT_MATCH - - def visit_false(self) -> bool: - # all rows fail - return ROWS_CANNOT_MATCH - - def visit_not(self, child_result: bool) -> bool: - raise ValueError(f"NOT should be rewritten: {child_result}") - - def visit_and(self, left_result: bool, right_result: bool) -> bool: - return left_result and right_result - - def visit_or(self, left_result: bool, right_result: bool) -> bool: - return left_result or right_result - def visit_is_null(self, term: BoundTerm[L]) -> bool: field_id = term.ref().field.field_id @@ -1421,3 +1431,303 @@ def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool return ROWS_CANNOT_MATCH return ROWS_MIGHT_MATCH + + +def strict_projection( + schema: Schema, spec: PartitionSpec, case_sensitive: bool = True +) -> Callable[[BooleanExpression], BooleanExpression]: + return StrictProjection(schema, spec, case_sensitive).project + + +class StrictProjection(ProjectionEvaluator): + def visit_bound_predicate(self, predicate: BoundPredicate[Any]) -> BooleanExpression: + parts = self.spec.fields_by_source_id(predicate.term.ref().field.field_id) + + result: BooleanExpression = AlwaysFalse() + for part in parts: + # consider (ts > 2019-01-01T01:00:00) with day(ts) and hour(ts) + # projections: d >= 2019-01-02 and h >= 2019-01-01-02 (note the inclusive bounds). + # any timestamp where either projection predicate is true must match the original + # predicate. For example, ts = 2019-01-01T03:00:00 matches the hour projection but not + # the day, but does match the original predicate. + strict_projection = part.transform.strict_project(name=part.name, pred=predicate) + if strict_projection is not None: + result = Or(result, strict_projection) + + return result + + +class _StrictMetricsEvaluator(_MetricsEvaluator): + struct: StructType + expr: BooleanExpression + + def __init__( + self, schema: Schema, expr: BooleanExpression, case_sensitive: bool = True, include_empty_files: bool = False + ) -> None: + self.struct = schema.as_struct() + self.include_empty_files = include_empty_files + self.expr = bind(schema, rewrite_not(expr), case_sensitive) + + def eval(self, file: DataFile) -> bool: + """Test whether all records within the file match the expression. + + Args: + file: A data file + + Returns: false if the file may contain any row that doesn't match + the expression, true otherwise. + """ + if file.record_count <= 0: + # Older version don't correctly implement record count from avro file and thus + # set record count -1 when importing avro tables to iceberg tables. This should + # be updated once we implemented and set correct record count. + return ROWS_MUST_MATCH + + self.value_counts = file.value_counts or EMPTY_DICT + self.null_counts = file.null_value_counts or EMPTY_DICT + self.nan_counts = file.nan_value_counts or EMPTY_DICT + self.lower_bounds = file.lower_bounds or EMPTY_DICT + self.upper_bounds = file.upper_bounds or EMPTY_DICT + + return visit(self.expr, self) + + def visit_is_null(self, term: BoundTerm[L]) -> bool: + # no need to check whether the field is required because binding evaluates that case + # if the column has any non-null values, the expression does not match + field_id = term.ref().field.field_id + + if self._contains_nulls_only(field_id): + return ROWS_MUST_MATCH + else: + return ROWS_MIGHT_NOT_MATCH + + def visit_not_null(self, term: BoundTerm[L]) -> bool: + # no need to check whether the field is required because binding evaluates that case + # if the column has any non-null values, the expression does not match + field_id = term.ref().field.field_id + + if (null_count := self.null_counts.get(field_id)) is not None and null_count == 0: + return ROWS_MUST_MATCH + else: + return ROWS_MIGHT_NOT_MATCH + + def visit_is_nan(self, term: BoundTerm[L]) -> bool: + field_id = term.ref().field.field_id + + if self._contains_nans_only(field_id): + return ROWS_MUST_MATCH + else: + return ROWS_MIGHT_NOT_MATCH + + def visit_not_nan(self, term: BoundTerm[L]) -> bool: + field_id = term.ref().field.field_id + + if (nan_count := self.nan_counts.get(field_id)) is not None and nan_count == 0: + return ROWS_MUST_MATCH + + if self._contains_nulls_only(field_id): + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_less_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when: <----------Min----Max---X-------> + + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + if upper_bytes := self.upper_bounds.get(field_id): + field = self._get_field(field_id) + upper = _from_byte_buffer(field.field_type, upper_bytes) + + if upper < literal.value: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_less_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when: <----------Min----Max---X-------> + + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + if upper_bytes := self.upper_bounds.get(field_id): + field = self._get_field(field_id) + upper = _from_byte_buffer(field.field_type, upper_bytes) + + if upper <= literal.value: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_greater_than(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when: <-------X---Min----Max----------> + + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + if lower_bytes := self.lower_bounds.get(field_id): + field = self._get_field(field_id) + lower = _from_byte_buffer(field.field_type, lower_bytes) + + if self._is_nan(lower): + # NaN indicates unreliable bounds. + # See the _StrictMetricsEvaluator docs for more. + return ROWS_MIGHT_NOT_MATCH + + if lower > literal.value: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_greater_than_or_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when: <-------X---Min----Max----------> + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + if lower_bytes := self.lower_bounds.get(field_id): + field = self._get_field(field_id) + lower = _from_byte_buffer(field.field_type, lower_bytes) + + if self._is_nan(lower): + # NaN indicates unreliable bounds. + # See the _StrictMetricsEvaluator docs for more. + return ROWS_MIGHT_NOT_MATCH + + if lower >= literal.value: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when Min == X == Max + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + if (lower_bytes := self.lower_bounds.get(field_id)) and (upper_bytes := self.upper_bounds.get(field_id)): + field = self._get_field(field_id) + lower = _from_byte_buffer(field.field_type, lower_bytes) + upper = _from_byte_buffer(field.field_type, upper_bytes) + + if lower != literal.value or upper != literal.value: + return ROWS_MIGHT_NOT_MATCH + else: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_not_equal(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + # Rows must match when X < Min or Max < X because it is not in the range + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MUST_MATCH + + field = self._get_field(field_id) + + if lower_bytes := self.lower_bounds.get(field_id): + lower = _from_byte_buffer(field.field_type, lower_bytes) + + if self._is_nan(lower): + # NaN indicates unreliable bounds. + # See the _StrictMetricsEvaluator docs for more. + return ROWS_MIGHT_NOT_MATCH + + if lower > literal.value: + return ROWS_MUST_MATCH + + if upper_bytes := self.upper_bounds.get(field_id): + upper = _from_byte_buffer(field.field_type, upper_bytes) + + if upper < literal.value: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MIGHT_NOT_MATCH + + field = self._get_field(field_id) + + if (lower_bytes := self.lower_bounds.get(field_id)) and (upper_bytes := self.upper_bounds.get(field_id)): + # similar to the implementation in eq, first check if the lower bound is in the set + lower = _from_byte_buffer(field.field_type, lower_bytes) + if lower not in literals: + return ROWS_MIGHT_NOT_MATCH + + # check if the upper bound is in the set + upper = _from_byte_buffer(field.field_type, upper_bytes) + if upper not in literals: + return ROWS_MIGHT_NOT_MATCH + + # finally check if the lower bound and the upper bound are equal + if lower != upper: + return ROWS_MIGHT_NOT_MATCH + + # All values must be in the set if the lower bound and the upper bound are + # in the set and are equal. + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_not_in(self, term: BoundTerm[L], literals: Set[L]) -> bool: + field_id = term.ref().field.field_id + + if self._can_contain_nulls(field_id) or self._can_contain_nans(field_id): + return ROWS_MUST_MATCH + + field = self._get_field(field_id) + + if lower_bytes := self.lower_bounds.get(field_id): + lower = _from_byte_buffer(field.field_type, lower_bytes) + + if self._is_nan(lower): + # NaN indicates unreliable bounds. + # See the StrictMetricsEvaluator docs for more. + return ROWS_MIGHT_NOT_MATCH + + literals = {val for val in literals if lower <= val} + if len(literals) == 0: + return ROWS_MUST_MATCH + + if upper_bytes := self.upper_bounds.get(field_id): + upper = _from_byte_buffer(field.field_type, upper_bytes) + + literals = {val for val in literals if upper >= val} + + if len(literals) == 0: + return ROWS_MUST_MATCH + + return ROWS_MIGHT_NOT_MATCH + + def visit_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return ROWS_MIGHT_NOT_MATCH + + def visit_not_starts_with(self, term: BoundTerm[L], literal: Literal[L]) -> bool: + return ROWS_MIGHT_NOT_MATCH + + def _get_field(self, field_id: int) -> NestedField: + field = self.struct.field(field_id=field_id) + if field is None: + raise ValueError(f"Cannot find field, might be nested or missing: {field_id}") + + return field + + def _can_contain_nulls(self, field_id: int) -> bool: + return (null_count := self.null_counts.get(field_id)) is not None and null_count > 0 + + def _can_contain_nans(self, field_id: int) -> bool: + return (nan_count := self.nan_counts.get(field_id)) is not None and nan_count > 0 diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 7f446e0638..738cd77bfd 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -69,8 +69,8 @@ ) from sortedcontainers import SortedList -from pyiceberg.avro.resolver import ResolveError from pyiceberg.conversions import to_bytes +from pyiceberg.exceptions import ResolveError from pyiceberg.expressions import ( AlwaysTrue, BooleanExpression, @@ -111,6 +111,7 @@ DataFileContent, FileFormat, ) +from pyiceberg.partitioning import PartitionField, PartitionSpec, partition_record_value from pyiceberg.schema import ( PartnerAccessor, PreOrderSchemaVisitor, @@ -945,13 +946,9 @@ def _task_to_table( projected_field_ids: Set[int], positional_deletes: Optional[List[ChunkedArray]], case_sensitive: bool, - row_counts: List[int], limit: Optional[int] = None, name_mapping: Optional[NameMapping] = None, ) -> Optional[pa.Table]: - if limit and sum(row_counts) >= limit: - return None - _, _, path = PyArrowFileIO.parse_location(task.file.file_path) arrow_format = ds.ParquetFileFormat(pre_buffer=True, buffer_size=(ONE_MEGABYTE * 8)) with fs.open_input_file(path) as fin: @@ -1014,11 +1011,6 @@ def _task_to_table( if len(arrow_table) < 1: return None - if limit is not None and sum(row_counts) >= limit: - return None - - row_counts.append(len(arrow_table)) - return to_requested_schema(projected_schema, file_project_schema, arrow_table) @@ -1084,7 +1076,6 @@ def project_table( id for id in projected_schema.field_ids if not isinstance(projected_schema.find_type(id), (MapType, ListType)) }.union(extract_field_ids(bound_row_filter)) - row_counts: List[int] = [] deletes_per_file = _read_all_delete_files(fs, tasks) executor = ExecutorFactory.get_or_create() futures = [ @@ -1097,21 +1088,21 @@ def project_table( projected_field_ids, deletes_per_file.get(task.file.file_path), case_sensitive, - row_counts, limit, table.name_mapping(), ) for task in tasks ] - + total_row_count = 0 # for consistent ordering, we need to maintain future order futures_index = {f: i for i, f in enumerate(futures)} completed_futures: SortedList[Future[pa.Table]] = SortedList(iterable=[], key=lambda f: futures_index[f]) for future in concurrent.futures.as_completed(futures): completed_futures.add(future) - + if table_result := future.result(): + total_row_count += len(table_result) # stop early if limit is satisfied - if limit is not None and sum(row_counts) >= limit: + if limit is not None and total_row_count >= limit: break # by now, we've either completed all tasks or satisfied the limit @@ -1594,29 +1585,88 @@ def parquet_path_to_id_mapping( return result -def fill_parquet_file_metadata( - data_file: DataFile, +@dataclass(frozen=True) +class DataFileStatistics: + record_count: int + column_sizes: Dict[int, int] + value_counts: Dict[int, int] + null_value_counts: Dict[int, int] + nan_value_counts: Dict[int, int] + column_aggregates: Dict[int, StatsAggregator] + split_offsets: List[int] + + def _partition_value(self, partition_field: PartitionField, schema: Schema) -> Any: + if partition_field.source_id not in self.column_aggregates: + return None + + if not partition_field.transform.preserves_order: + raise ValueError( + f"Cannot infer partition value from parquet metadata for a non-linear Partition Field: {partition_field.name} with transform {partition_field.transform}" + ) + + lower_value = partition_record_value( + partition_field=partition_field, + value=self.column_aggregates[partition_field.source_id].current_min, + schema=schema, + ) + upper_value = partition_record_value( + partition_field=partition_field, + value=self.column_aggregates[partition_field.source_id].current_max, + schema=schema, + ) + if lower_value != upper_value: + raise ValueError( + f"Cannot infer partition value from parquet metadata as there are more than one partition values for Partition Field: {partition_field.name}. {lower_value=}, {upper_value=}" + ) + return lower_value + + def partition(self, partition_spec: PartitionSpec, schema: Schema) -> Record: + return Record(**{field.name: self._partition_value(field, schema) for field in partition_spec.fields}) + + def to_serialized_dict(self) -> Dict[str, Any]: + lower_bounds = {} + upper_bounds = {} + + for k, agg in self.column_aggregates.items(): + _min = agg.min_as_bytes() + if _min is not None: + lower_bounds[k] = _min + _max = agg.max_as_bytes() + if _max is not None: + upper_bounds[k] = _max + return { + "record_count": self.record_count, + "column_sizes": self.column_sizes, + "value_counts": self.value_counts, + "null_value_counts": self.null_value_counts, + "nan_value_counts": self.nan_value_counts, + "lower_bounds": lower_bounds, + "upper_bounds": upper_bounds, + "split_offsets": self.split_offsets, + } + + +def data_file_statistics_from_parquet_metadata( parquet_metadata: pq.FileMetaData, stats_columns: Dict[int, StatisticsCollector], parquet_column_mapping: Dict[str, int], -) -> None: +) -> DataFileStatistics: """ - Compute and fill the following fields of the DataFile object. + Compute and return DataFileStatistics that includes the following. - - file_format + - record_count - column_sizes - value_counts - null_value_counts - nan_value_counts - - lower_bounds - - upper_bounds + - column_aggregates - split_offsets Args: - data_file (DataFile): A DataFile object representing the Parquet file for which metadata is to be filled. parquet_metadata (pyarrow.parquet.FileMetaData): A pyarrow metadata object. stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to set the mode for column metrics collection + parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID """ if parquet_metadata.num_columns != len(stats_columns): raise ValueError( @@ -1695,35 +1745,24 @@ def fill_parquet_file_metadata( split_offsets.sort() - lower_bounds = {} - upper_bounds = {} - - for k, agg in col_aggs.items(): - _min = agg.min_as_bytes() - if _min is not None: - lower_bounds[k] = _min - _max = agg.max_as_bytes() - if _max is not None: - upper_bounds[k] = _max - for field_id in invalidate_col: - del lower_bounds[field_id] - del upper_bounds[field_id] + del col_aggs[field_id] del null_value_counts[field_id] - data_file.record_count = parquet_metadata.num_rows - data_file.column_sizes = column_sizes - data_file.value_counts = value_counts - data_file.null_value_counts = null_value_counts - data_file.nan_value_counts = nan_value_counts - data_file.lower_bounds = lower_bounds - data_file.upper_bounds = upper_bounds - data_file.split_offsets = split_offsets + return DataFileStatistics( + record_count=parquet_metadata.num_rows, + column_sizes=column_sizes, + value_counts=value_counts, + null_value_counts=null_value_counts, + nan_value_counts=nan_value_counts, + column_aggregates=col_aggs, + split_offsets=split_offsets, + ) def write_file(io: FileIO, table_metadata: TableMetadata, tasks: Iterator[WriteTask]) -> Iterator[DataFile]: schema = table_metadata.schema() - arrow_file_schema = schema_to_pyarrow(schema) + arrow_file_schema = schema.as_arrow() parquet_writer_kwargs = _get_parquet_writer_kwargs(table_metadata.properties) row_group_size = PropertyUtil.property_as_int( @@ -1739,6 +1778,11 @@ def write_parquet(task: WriteTask) -> DataFile: with pq.ParquetWriter(fos, schema=arrow_file_schema, **parquet_writer_kwargs) as writer: writer.write(pa.Table.from_batches(task.record_batches), row_group_size=row_group_size) + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=writer.writer.metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), + ) data_file = DataFile( content=DataFileContent.DATA, file_path=file_path, @@ -1753,13 +1797,9 @@ def write_parquet(task: WriteTask) -> DataFile: spec_id=table_metadata.default_spec_id, equality_ids=None, key_metadata=None, + **statistics.to_serialized_dict(), ) - fill_parquet_file_metadata( - data_file=data_file, - parquet_metadata=writer.writer.metadata, - stats_columns=compute_statistics_plan(schema, table_metadata.properties), - parquet_column_mapping=parquet_path_to_id_mapping(schema), - ) + return data_file executor = ExecutorFactory.get_or_create() @@ -1784,6 +1824,38 @@ def bin_pack_arrow_table(tbl: pa.Table, target_file_size: int) -> Iterator[List[ return bin_packed_record_batches +def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]: + for file_path in file_paths: + input_file = io.new_input(file_path) + with input_file.open() as input_stream: + parquet_metadata = pq.read_metadata(input_stream) + + if visit_pyarrow(parquet_metadata.schema.to_arrow_schema(), _HasIds()): + raise NotImplementedError( + f"Cannot add file {file_path} because it has field IDs. `add_files` only supports addition of files without field_ids" + ) + schema = table_metadata.schema() + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=parquet_metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), + ) + data_file = DataFile( + content=DataFileContent.DATA, + file_path=file_path, + file_format=FileFormat.PARQUET, + partition=statistics.partition(table_metadata.spec(), table_metadata.schema()), + file_size_in_bytes=len(input_file), + sort_order_id=None, + spec_id=table_metadata.default_spec_id, + equality_ids=None, + key_metadata=None, + **statistics.to_serialized_dict(), + ) + + yield data_file + + ICEBERG_UNCOMPRESSED_CODEC = "uncompressed" PYARROW_UNCOMPRESSED_CODEC = "none" diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 0504626d07..03dc3199bf 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -308,6 +308,7 @@ def data_file_with_partition(partition_type: StructType, format_version: Literal field_id=field.field_id, name=field.name, field_type=partition_field_to_data_file_partition_field(field.field_type), + required=field.required, ) for field in partition_type.fields ]) diff --git a/pyiceberg/partitioning.py b/pyiceberg/partitioning.py index 6fa0286282..16f158828d 100644 --- a/pyiceberg/partitioning.py +++ b/pyiceberg/partitioning.py @@ -218,7 +218,8 @@ def partition_type(self, schema: Schema) -> StructType: for field in self.fields: source_type = schema.find_type(field.source_id) result_type = field.transform.result_type(source_type) - nested_fields.append(NestedField(field.field_id, field.name, result_type, required=False)) + required = schema.find_field(field.source_id).required + nested_fields.append(NestedField(field.field_id, field.name, result_type, required=required)) return StructType(*nested_fields) def partition_to_path(self, data: Record, schema: Schema) -> str: @@ -387,16 +388,33 @@ def partition(self) -> Record: # partition key transformed with iceberg interna if len(partition_fields) != 1: raise ValueError("partition_fields must contain exactly one field.") partition_field = partition_fields[0] - iceberg_type = self.schema.find_field(name_or_id=raw_partition_field_value.field.source_id).field_type - iceberg_typed_value = _to_partition_representation(iceberg_type, raw_partition_field_value.value) - transformed_value = partition_field.transform.transform(iceberg_type)(iceberg_typed_value) - iceberg_typed_key_values[partition_field.name] = transformed_value + iceberg_typed_key_values[partition_field.name] = partition_record_value( + partition_field=partition_field, + value=raw_partition_field_value.value, + schema=self.schema, + ) return Record(**iceberg_typed_key_values) def to_path(self) -> str: return self.partition_spec.partition_to_path(self.partition, self.schema) +def partition_record_value(partition_field: PartitionField, value: Any, schema: Schema) -> Any: + """ + Return the Partition Record representation of the value. + + The value is first converted to internal partition representation. + For example, UUID is converted to bytes[16], DateType to days since epoch, etc. + + Then the corresponding PartitionField's transform is applied to return + the final partition record value. + """ + iceberg_type = schema.find_field(name_or_id=partition_field.source_id).field_type + iceberg_typed_value = _to_partition_representation(iceberg_type, value) + transformed_value = partition_field.transform.transform(iceberg_type)(iceberg_typed_value) + return transformed_value + + @singledispatch def _to_partition_representation(type: IcebergType, value: Any) -> Any: return TypeError(f"Unsupported partition field type: {type}") diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py index e805895a7b..b2739d8618 100644 --- a/pyiceberg/schema.py +++ b/pyiceberg/schema.py @@ -64,6 +64,8 @@ ) if TYPE_CHECKING: + import pyarrow as pa + from pyiceberg.table.name_mapping import ( NameMapping, ) @@ -180,6 +182,12 @@ def as_struct(self) -> StructType: """Return the schema as a struct.""" return StructType(*self.fields) + def as_arrow(self) -> "pa.Schema": + """Return the schema as an Arrow schema.""" + from pyiceberg.io.pyarrow import schema_to_pyarrow + + return schema_to_pyarrow(self) + def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> NestedField: """Find a field using a field name or field ID. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 411318ca87..18fac99312 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -42,7 +42,7 @@ Union, ) -from pydantic import Field, SerializeAsAny, field_validator +from pydantic import Field, field_validator from sortedcontainers import SortedList from typing_extensions import Annotated @@ -71,7 +71,6 @@ from pyiceberg.partitioning import ( INITIAL_PARTITION_SPEC_ID, PARTITION_FIELD_ID_START, - IdentityTransform, PartitionField, PartitionSpec, _PartitionNameGenerator, @@ -108,7 +107,7 @@ update_snapshot_summaries, ) from pyiceberg.table.sorting import SortOrder -from pyiceberg.transforms import TimeTransform, Transform, VoidTransform +from pyiceberg.transforms import IdentityTransform, TimeTransform, Transform, VoidTransform from pyiceberg.typedef import ( EMPTY_DICT, IcebergBaseModel, @@ -116,6 +115,7 @@ Identifier, KeyDefaultDict, Properties, + Record, ) from pyiceberg.types import ( IcebergType, @@ -145,7 +145,15 @@ _JAVA_LONG_MAX = 9223372036854775807 -def _check_schema(table_schema: Schema, other_schema: "pa.Schema") -> None: +def _check_schema_compatible(table_schema: Schema, other_schema: "pa.Schema") -> None: + """ + Check if the `table_schema` is compatible with `other_schema`. + + Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type. + + Raises: + ValueError: If the schemas are not compatible. + """ from pyiceberg.io.pyarrow import _pyarrow_to_schema_without_ids, pyarrow_to_schema name_mapping = table_schema.name_mapping @@ -215,6 +223,9 @@ class TableProperties: METRICS_MODE_COLUMN_CONF_PREFIX = "write.metadata.metrics.column" + WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit" + WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0 + DEFAULT_NAME_MAPPING = "schema.name-mapping.default" FORMAT_VERSION = "format-version" DEFAULT_FORMAT_VERSION = 2 @@ -324,15 +335,20 @@ def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive Returns: A new UpdateSchema. """ - return UpdateSchema(self, allow_incompatible_changes=allow_incompatible_changes, case_sensitive=case_sensitive) + return UpdateSchema( + self, + allow_incompatible_changes=allow_incompatible_changes, + case_sensitive=case_sensitive, + name_mapping=self._table.name_mapping(), + ) - def update_snapshot(self) -> UpdateSnapshot: + def update_snapshot(self, snapshot_properties: Dict[str, str] = EMPTY_DICT) -> UpdateSnapshot: """Create a new UpdateSnapshot to produce a new snapshot for the table. Returns: A new UpdateSnapshot """ - return UpdateSnapshot(self, io=self._table.io) + return UpdateSnapshot(self, io=self._table.io, snapshot_properties=snapshot_properties) def update_spec(self) -> UpdateSpec: """Create a new UpdateSpec to update the partitioning of the table. @@ -380,77 +396,56 @@ def commit_transaction(self) -> Table: return self._table -class TableUpdateAction(Enum): - upgrade_format_version = "upgrade-format-version" - add_schema = "add-schema" - set_current_schema = "set-current-schema" - add_spec = "add-spec" - set_default_spec = "set-default-spec" - add_sort_order = "add-sort-order" - set_default_sort_order = "set-default-sort-order" - add_snapshot = "add-snapshot" - set_snapshot_ref = "set-snapshot-ref" - remove_snapshots = "remove-snapshots" - remove_snapshot_ref = "remove-snapshot-ref" - set_location = "set-location" - set_properties = "set-properties" - remove_properties = "remove-properties" - - -class TableUpdate(IcebergBaseModel): - action: TableUpdateAction - - -class UpgradeFormatVersionUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.upgrade_format_version +class UpgradeFormatVersionUpdate(IcebergBaseModel): + action: Literal['upgrade-format-version'] = Field(default="upgrade-format-version") format_version: int = Field(alias="format-version") -class AddSchemaUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.add_schema +class AddSchemaUpdate(IcebergBaseModel): + action: Literal['add-schema'] = Field(default="add-schema") schema_: Schema = Field(alias="schema") # This field is required: https://github.com/apache/iceberg/pull/7445 last_column_id: int = Field(alias="last-column-id") -class SetCurrentSchemaUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_current_schema +class SetCurrentSchemaUpdate(IcebergBaseModel): + action: Literal['set-current-schema'] = Field(default="set-current-schema") schema_id: int = Field( alias="schema-id", description="Schema ID to set as current, or -1 to set last added schema", default=-1 ) -class AddPartitionSpecUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.add_spec +class AddPartitionSpecUpdate(IcebergBaseModel): + action: Literal['add-spec'] = Field(default="add-spec") spec: PartitionSpec -class SetDefaultSpecUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_default_spec +class SetDefaultSpecUpdate(IcebergBaseModel): + action: Literal['set-default-spec'] = Field(default="set-default-spec") spec_id: int = Field( alias="spec-id", description="Partition spec ID to set as the default, or -1 to set last added spec", default=-1 ) -class AddSortOrderUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.add_sort_order +class AddSortOrderUpdate(IcebergBaseModel): + action: Literal['add-sort-order'] = Field(default="add-sort-order") sort_order: SortOrder = Field(alias="sort-order") -class SetDefaultSortOrderUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_default_sort_order +class SetDefaultSortOrderUpdate(IcebergBaseModel): + action: Literal['set-default-sort-order'] = Field(default="set-default-sort-order") sort_order_id: int = Field( alias="sort-order-id", description="Sort order ID to set as the default, or -1 to set last added sort order", default=-1 ) -class AddSnapshotUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.add_snapshot +class AddSnapshotUpdate(IcebergBaseModel): + action: Literal['add-snapshot'] = Field(default="add-snapshot") snapshot: Snapshot -class SetSnapshotRefUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_snapshot_ref +class SetSnapshotRefUpdate(IcebergBaseModel): + action: Literal['set-snapshot-ref'] = Field(default="set-snapshot-ref") ref_name: str = Field(alias="ref-name") type: Literal["tag", "branch"] snapshot_id: int = Field(alias="snapshot-id") @@ -459,23 +454,23 @@ class SetSnapshotRefUpdate(TableUpdate): min_snapshots_to_keep: Annotated[Optional[int], Field(alias="min-snapshots-to-keep", default=None)] -class RemoveSnapshotsUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.remove_snapshots +class RemoveSnapshotsUpdate(IcebergBaseModel): + action: Literal['remove-snapshots'] = Field(default="remove-snapshots") snapshot_ids: List[int] = Field(alias="snapshot-ids") -class RemoveSnapshotRefUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.remove_snapshot_ref +class RemoveSnapshotRefUpdate(IcebergBaseModel): + action: Literal['remove-snapshot-ref'] = Field(default="remove-snapshot-ref") ref_name: str = Field(alias="ref-name") -class SetLocationUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_location +class SetLocationUpdate(IcebergBaseModel): + action: Literal['set-location'] = Field(default="set-location") location: str -class SetPropertiesUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.set_properties +class SetPropertiesUpdate(IcebergBaseModel): + action: Literal['set-properties'] = Field(default="set-properties") updates: Dict[str, str] @field_validator('updates', mode='before') @@ -483,11 +478,32 @@ def transform_properties_dict_value_to_str(cls, properties: Properties) -> Dict[ return transform_dict_value_to_str(properties) -class RemovePropertiesUpdate(TableUpdate): - action: TableUpdateAction = TableUpdateAction.remove_properties +class RemovePropertiesUpdate(IcebergBaseModel): + action: Literal['remove-properties'] = Field(default="remove-properties") removals: List[str] +TableUpdate = Annotated[ + Union[ + UpgradeFormatVersionUpdate, + AddSchemaUpdate, + SetCurrentSchemaUpdate, + AddPartitionSpecUpdate, + SetDefaultSpecUpdate, + AddSortOrderUpdate, + SetDefaultSortOrderUpdate, + AddSnapshotUpdate, + SetSnapshotRefUpdate, + RemoveSnapshotsUpdate, + RemoveSnapshotRefUpdate, + SetLocationUpdate, + SetPropertiesUpdate, + RemovePropertiesUpdate, + ], + Field(discriminator='action'), +] + + class _TableMetadataUpdateContext: _updates: List[TableUpdate] @@ -499,21 +515,15 @@ def add_update(self, update: TableUpdate) -> None: def is_added_snapshot(self, snapshot_id: int) -> bool: return any( - update.snapshot.snapshot_id == snapshot_id - for update in self._updates - if update.action == TableUpdateAction.add_snapshot + update.snapshot.snapshot_id == snapshot_id for update in self._updates if isinstance(update, AddSnapshotUpdate) ) def is_added_schema(self, schema_id: int) -> bool: - return any( - update.schema_.schema_id == schema_id for update in self._updates if update.action == TableUpdateAction.add_schema - ) + return any(update.schema_.schema_id == schema_id for update in self._updates if isinstance(update, AddSchemaUpdate)) def is_added_sort_order(self, sort_order_id: int) -> bool: return any( - update.sort_order.order_id == sort_order_id - for update in self._updates - if update.action == TableUpdateAction.add_sort_order + update.sort_order.order_id == sort_order_id for update in self._updates if isinstance(update, AddSortOrderUpdate) ) @@ -764,7 +774,7 @@ def update_table_metadata(base_metadata: TableMetadata, updates: Tuple[TableUpda return new_metadata.model_copy(deep=True) -class TableRequirement(IcebergBaseModel): +class ValidatableTableRequirement(IcebergBaseModel): type: str @abstractmethod @@ -780,7 +790,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ... -class AssertCreate(TableRequirement): +class AssertCreate(ValidatableTableRequirement): """The table must not already exist; used for create transactions.""" type: Literal["assert-create"] = Field(default="assert-create") @@ -790,7 +800,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: raise CommitFailedException("Table already exists") -class AssertTableUUID(TableRequirement): +class AssertTableUUID(ValidatableTableRequirement): """The table UUID must match the requirement's `uuid`.""" type: Literal["assert-table-uuid"] = Field(default="assert-table-uuid") @@ -803,7 +813,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: raise CommitFailedException(f"Table UUID does not match: {self.uuid} != {base_metadata.table_uuid}") -class AssertRefSnapshotId(TableRequirement): +class AssertRefSnapshotId(ValidatableTableRequirement): """The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`. if `snapshot-id` is `null` or missing, the ref must not already exist. @@ -828,7 +838,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: raise CommitFailedException(f"Requirement failed: branch or tag {self.ref} is missing, expected {self.snapshot_id}") -class AssertLastAssignedFieldId(TableRequirement): +class AssertLastAssignedFieldId(ValidatableTableRequirement): """The table's last assigned column id must match the requirement's `last-assigned-field-id`.""" type: Literal["assert-last-assigned-field-id"] = Field(default="assert-last-assigned-field-id") @@ -843,7 +853,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ) -class AssertCurrentSchemaId(TableRequirement): +class AssertCurrentSchemaId(ValidatableTableRequirement): """The table's current schema id must match the requirement's `current-schema-id`.""" type: Literal["assert-current-schema-id"] = Field(default="assert-current-schema-id") @@ -858,7 +868,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ) -class AssertLastAssignedPartitionId(TableRequirement): +class AssertLastAssignedPartitionId(ValidatableTableRequirement): """The table's last assigned partition id must match the requirement's `last-assigned-partition-id`.""" type: Literal["assert-last-assigned-partition-id"] = Field(default="assert-last-assigned-partition-id") @@ -873,7 +883,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ) -class AssertDefaultSpecId(TableRequirement): +class AssertDefaultSpecId(ValidatableTableRequirement): """The table's default spec id must match the requirement's `default-spec-id`.""" type: Literal["assert-default-spec-id"] = Field(default="assert-default-spec-id") @@ -888,7 +898,7 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ) -class AssertDefaultSortOrderId(TableRequirement): +class AssertDefaultSortOrderId(ValidatableTableRequirement): """The table's default sort order id must match the requirement's `default-sort-order-id`.""" type: Literal["assert-default-sort-order-id"] = Field(default="assert-default-sort-order-id") @@ -903,6 +913,20 @@ def validate(self, base_metadata: Optional[TableMetadata]) -> None: ) +TableRequirement = Annotated[ + Union[ + AssertCreate, + AssertTableUUID, + AssertRefSnapshotId, + AssertLastAssignedFieldId, + AssertCurrentSchemaId, + AssertLastAssignedPartitionId, + AssertDefaultSpecId, + AssertDefaultSortOrderId, + ], + Field(discriminator='type'), +] + UpdatesAndRequirements = Tuple[Tuple[TableUpdate, ...], Tuple[TableRequirement, ...]] @@ -924,8 +948,8 @@ class TableIdentifier(IcebergBaseModel): class CommitTableRequest(IcebergBaseModel): identifier: TableIdentifier = Field() - requirements: Tuple[SerializeAsAny[TableRequirement], ...] = Field(default_factory=tuple) - updates: Tuple[SerializeAsAny[TableUpdate], ...] = Field(default_factory=tuple) + requirements: Tuple[TableRequirement, ...] = Field(default_factory=tuple) + updates: Tuple[TableUpdate, ...] = Field(default_factory=tuple) class CommitTableResponse(IcebergBaseModel): @@ -957,6 +981,11 @@ def transaction(self) -> Transaction: """ return Transaction(self) + @property + def inspect(self) -> InspectTable: + """Return the InspectTable object to browse the table metadata.""" + return InspectTable(self) + def refresh(self) -> Table: """Refresh the current table metadata.""" fresh = self.catalog.load_table(self.identifier[1:]) @@ -1081,12 +1110,13 @@ def name_mapping(self) -> Optional[NameMapping]: else: return None - def append(self, df: pa.Table) -> None: + def append(self, df: pa.Table, snapshot_properties: Dict[str, str] = EMPTY_DICT) -> None: """ Shorthand API for appending a PyArrow table to the table. Args: df: The Arrow dataframe that will be appended to overwrite the table + snapshot_properties: Custom properties to be added to the snapshot summary """ try: import pyarrow as pa @@ -1099,10 +1129,13 @@ def append(self, df: pa.Table) -> None: if len(self.spec().fields) > 0: raise ValueError("Cannot write to partitioned tables") - _check_schema(self.schema(), other_schema=df.schema) + _check_schema_compatible(self.schema(), other_schema=df.schema) + # cast if the two schemas are compatible but not equal + if self.schema().as_arrow() != df.schema: + df = df.cast(self.schema().as_arrow()) with self.transaction() as txn: - with txn.update_snapshot().fast_append() as update_snapshot: + with txn.update_snapshot(snapshot_properties=snapshot_properties).fast_append() as update_snapshot: # skip writing data files if the dataframe is empty if df.shape[0] > 0: data_files = _dataframe_to_data_files( @@ -1111,7 +1144,9 @@ def append(self, df: pa.Table) -> None: for data_file in data_files: update_snapshot.append_data_file(data_file) - def overwrite(self, df: pa.Table, overwrite_filter: BooleanExpression = ALWAYS_TRUE) -> None: + def overwrite( + self, df: pa.Table, overwrite_filter: BooleanExpression = ALWAYS_TRUE, snapshot_properties: Dict[str, str] = EMPTY_DICT + ) -> None: """ Shorthand for overwriting the table with a PyArrow table. @@ -1119,6 +1154,7 @@ def overwrite(self, df: pa.Table, overwrite_filter: BooleanExpression = ALWAYS_T df: The Arrow dataframe that will be used to overwrite the table overwrite_filter: ALWAYS_TRUE when you overwrite all the data, or a boolean expression in case of a partial overwrite + snapshot_properties: Custom properties to be added to the snapshot summary """ try: import pyarrow as pa @@ -1134,10 +1170,13 @@ def overwrite(self, df: pa.Table, overwrite_filter: BooleanExpression = ALWAYS_T if len(self.spec().fields) > 0: raise ValueError("Cannot write to partitioned tables") - _check_schema(self.schema(), other_schema=df.schema) + _check_schema_compatible(self.schema(), other_schema=df.schema) + # cast if the two schemas are compatible but not equal + if self.schema().as_arrow() != df.schema: + df = df.cast(self.schema().as_arrow()) with self.transaction() as txn: - with txn.update_snapshot().overwrite() as update_snapshot: + with txn.update_snapshot(snapshot_properties=snapshot_properties).overwrite() as update_snapshot: # skip writing data files if the dataframe is empty if df.shape[0] > 0: data_files = _dataframe_to_data_files( @@ -1146,6 +1185,24 @@ def overwrite(self, df: pa.Table, overwrite_filter: BooleanExpression = ALWAYS_T for data_file in data_files: update_snapshot.append_data_file(data_file) + def add_files(self, file_paths: List[str]) -> None: + """ + Shorthand API for adding files as data files to the table. + + Args: + file_paths: The list of full file paths to be added as data files to the table + + Raises: + FileNotFoundError: If the file does not exist. + """ + with self.transaction() as tx: + if self.name_mapping() is None: + tx.set_properties(**{TableProperties.DEFAULT_NAME_MAPPING: self.schema().name_mapping.model_dump_json()}) + with tx.update_snapshot().fast_append() as update_snapshot: + data_files = _parquet_files_to_data_files(table_metadata=self.metadata, file_paths=file_paths, io=self.io) + for data_file in data_files: + update_snapshot.append_data_file(data_file) + def update_spec(self, case_sensitive: bool = True) -> UpdateSpec: return UpdateSpec(Transaction(self, autocommit=True), case_sensitive=case_sensitive) @@ -2443,6 +2500,12 @@ def generate_data_file_filename(self, extension: str) -> str: return f"00000-{self.task_id}-{self.write_uuid}.{extension}" +@dataclass(frozen=True) +class AddFileTask: + file_path: str + partition_field_value: Record + + def _new_manifest_path(location: str, num: int, commit_uuid: uuid.UUID) -> str: return f'{location}/metadata/{commit_uuid}-m{num}.avro' @@ -2484,6 +2547,17 @@ def _dataframe_to_data_files( ) +def _parquet_files_to_data_files(table_metadata: TableMetadata, file_paths: List[str], io: FileIO) -> Iterable[DataFile]: + """Convert a list files into DataFiles. + + Returns: + An iterable that supplies DataFiles that describe the parquet files. + """ + from pyiceberg.io.pyarrow import parquet_files_to_data_files + + yield from parquet_files_to_data_files(io=io, table_metadata=table_metadata, file_paths=iter(file_paths)) + + class _MergingSnapshotProducer(UpdateTableMetadata["_MergingSnapshotProducer"]): commit_uuid: uuid.UUID _operation: Operation @@ -2497,6 +2571,7 @@ def __init__( transaction: Transaction, io: FileIO, commit_uuid: Optional[uuid.UUID] = None, + snapshot_properties: Dict[str, str] = EMPTY_DICT, ) -> None: super().__init__(transaction) self.commit_uuid = commit_uuid or uuid.uuid4() @@ -2508,6 +2583,7 @@ def __init__( snapshot.snapshot_id if (snapshot := self._transaction.table_metadata.current_snapshot()) else None ) self._added_data_files = [] + self.snapshot_properties = snapshot_properties def append_data_file(self, data_file: DataFile) -> _MergingSnapshotProducer: self._added_data_files.append(data_file) @@ -2575,11 +2651,21 @@ def _write_delete_manifest() -> List[ManifestFile]: return added_manifests.result() + delete_manifests.result() + existing_manifests.result() - def _summary(self) -> Summary: + def _summary(self, snapshot_properties: Dict[str, str] = EMPTY_DICT) -> Summary: ssc = SnapshotSummaryCollector() + partition_summary_limit = int( + self._transaction.table_metadata.properties.get( + TableProperties.WRITE_PARTITION_SUMMARY_LIMIT, TableProperties.WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT + ) + ) + ssc.set_partition_summary_limit(partition_summary_limit) for data_file in self._added_data_files: - ssc.add_file(data_file=data_file) + ssc.add_file( + data_file=data_file, + partition_spec=self._transaction.table_metadata.spec(), + schema=self._transaction.table_metadata.schema(), + ) previous_snapshot = ( self._transaction.table_metadata.snapshot_by_id(self._parent_snapshot_id) @@ -2588,7 +2674,7 @@ def _summary(self) -> Summary: ) return update_snapshot_summaries( - summary=Summary(operation=self._operation, **ssc.build()), + summary=Summary(operation=self._operation, **ssc.build(), **snapshot_properties), previous_summary=previous_snapshot.summary if previous_snapshot is not None else None, truncate_full_table=self._operation == Operation.OVERWRITE, ) @@ -2597,7 +2683,7 @@ def _commit(self) -> UpdatesAndRequirements: new_manifests = self._manifests() next_sequence_number = self._transaction.table_metadata.next_sequence_number() - summary = self._summary() + summary = self._summary(self.snapshot_properties) manifest_list_file_path = _generate_manifest_list_path( location=self._transaction.table_metadata.location, @@ -2712,13 +2798,17 @@ def _get_entries(manifest: ManifestFile) -> List[ManifestEntry]: class UpdateSnapshot: _transaction: Transaction _io: FileIO + _snapshot_properties: Dict[str, str] - def __init__(self, transaction: Transaction, io: FileIO) -> None: + def __init__(self, transaction: Transaction, io: FileIO, snapshot_properties: Dict[str, str]) -> None: self._transaction = transaction self._io = io + self._snapshot_properties = snapshot_properties def fast_append(self) -> FastAppendFiles: - return FastAppendFiles(operation=Operation.APPEND, transaction=self._transaction, io=self._io) + return FastAppendFiles( + operation=Operation.APPEND, transaction=self._transaction, io=self._io, snapshot_properties=self._snapshot_properties + ) def overwrite(self) -> OverwriteFiles: return OverwriteFiles( @@ -2727,6 +2817,7 @@ def overwrite(self) -> OverwriteFiles: else Operation.APPEND, transaction=self._transaction, io=self._io, + snapshot_properties=self._snapshot_properties, ) @@ -2971,3 +3062,49 @@ def _new_field_id(self) -> int: def _is_duplicate_partition(self, transform: Transform[Any, Any], partition_field: PartitionField) -> bool: return partition_field.field_id not in self._deletes and partition_field.transform == transform + + +class InspectTable: + tbl: Table + + def __init__(self, tbl: Table) -> None: + self.tbl = tbl + + try: + import pyarrow as pa # noqa + except ModuleNotFoundError as e: + raise ModuleNotFoundError("For metadata operations PyArrow needs to be installed") from e + + def snapshots(self) -> "pa.Table": + import pyarrow as pa + + snapshots_schema = pa.schema([ + pa.field('committed_at', pa.timestamp(unit='ms'), nullable=False), + pa.field('snapshot_id', pa.int64(), nullable=False), + pa.field('parent_id', pa.int64(), nullable=True), + pa.field('operation', pa.string(), nullable=True), + pa.field('manifest_list', pa.string(), nullable=False), + pa.field('summary', pa.map_(pa.string(), pa.string()), nullable=True), + ]) + snapshots = [] + for snapshot in self.tbl.metadata.snapshots: + if summary := snapshot.summary: + operation = summary.operation.value + additional_properties = snapshot.summary.additional_properties + else: + operation = None + additional_properties = None + + snapshots.append({ + 'committed_at': datetime.datetime.utcfromtimestamp(snapshot.timestamp_ms / 1000.0), + 'snapshot_id': snapshot.snapshot_id, + 'parent_id': snapshot.parent_snapshot_id, + 'operation': str(operation), + 'manifest_list': snapshot.manifest_list, + 'summary': additional_properties, + }) + + return pa.Table.from_pylist( + snapshots, + schema=snapshots_schema, + ) diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 323f6d85a8..3e1acf95f1 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -366,6 +366,11 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: fields = data[PARTITION_SPEC] data[PARTITION_SPECS] = [{SPEC_ID: INITIAL_SPEC_ID, FIELDS: fields}] data[DEFAULT_SPEC_ID] = INITIAL_SPEC_ID + elif data.get("partition_spec") is not None: + # Promote the spec from partition_spec to partition-specs + fields = data["partition_spec"] + data[PARTITION_SPECS] = [{SPEC_ID: INITIAL_SPEC_ID, FIELDS: fields}] + data[DEFAULT_SPEC_ID] = INITIAL_SPEC_ID else: data[PARTITION_SPECS] = [{"field-id": 0, "fields": ()}] @@ -389,7 +394,7 @@ def set_sort_orders(cls, data: Dict[str, Any]) -> Dict[str, Any]: Returns: The TableMetadata with the sort_orders set, if not provided. """ - if not data.get(SORT_ORDERS): + if not data.get(SORT_ORDERS) and not data.get("sort_orders"): data[SORT_ORDERS] = [UNSORTED_SORT_ORDER] return data diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index a2f15d4405..f74ac4b7d4 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -15,19 +15,16 @@ # specific language governing permissions and limitations # under the License. import time +from collections import defaultdict from enum import Enum -from typing import ( - Any, - Dict, - List, - Mapping, - Optional, -) +from typing import Any, DefaultDict, Dict, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer from pyiceberg.io import FileIO from pyiceberg.manifest import DataFile, DataFileContent, ManifestFile, read_manifest_list +from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.schema import Schema from pyiceberg.typedef import IcebergBaseModel ADDED_DATA_FILES = 'added-data-files' @@ -52,8 +49,8 @@ TOTAL_DELETE_FILES = 'total-delete-files' TOTAL_RECORDS = 'total-records' TOTAL_FILE_SIZE = 'total-files-size' - - +CHANGED_PARTITION_COUNT_PROP = 'changed-partition-count' +CHANGED_PARTITION_PREFIX = "partitions." OPERATION = "operation" @@ -77,6 +74,97 @@ def __repr__(self) -> str: return f"Operation.{self.name}" +class UpdateMetrics: + added_file_size: int + removed_file_size: int + added_data_files: int + removed_data_files: int + added_eq_delete_files: int + removed_eq_delete_files: int + added_pos_delete_files: int + removed_pos_delete_files: int + added_delete_files: int + removed_delete_files: int + added_records: int + deleted_records: int + added_pos_deletes: int + removed_pos_deletes: int + added_eq_deletes: int + removed_eq_deletes: int + + def __init__(self) -> None: + self.added_file_size = 0 + self.removed_file_size = 0 + self.added_data_files = 0 + self.removed_data_files = 0 + self.added_eq_delete_files = 0 + self.removed_eq_delete_files = 0 + self.added_pos_delete_files = 0 + self.removed_pos_delete_files = 0 + self.added_delete_files = 0 + self.removed_delete_files = 0 + self.added_records = 0 + self.deleted_records = 0 + self.added_pos_deletes = 0 + self.removed_pos_deletes = 0 + self.added_eq_deletes = 0 + self.removed_eq_deletes = 0 + + def add_file(self, data_file: DataFile) -> None: + self.added_file_size += data_file.file_size_in_bytes + + if data_file.content == DataFileContent.DATA: + self.added_data_files += 1 + self.added_records += data_file.record_count + elif data_file.content == DataFileContent.POSITION_DELETES: + self.added_delete_files += 1 + self.added_pos_delete_files += 1 + self.added_pos_deletes += data_file.record_count + elif data_file.content == DataFileContent.EQUALITY_DELETES: + self.added_delete_files += 1 + self.added_eq_delete_files += 1 + self.added_eq_deletes += data_file.record_count + else: + raise ValueError(f"Unknown data file content: {data_file.content}") + + def remove_file(self, data_file: DataFile) -> None: + self.removed_file_size += data_file.file_size_in_bytes + + if data_file.content == DataFileContent.DATA: + self.removed_data_files += 1 + self.deleted_records += data_file.record_count + elif data_file.content == DataFileContent.POSITION_DELETES: + self.removed_delete_files += 1 + self.removed_pos_delete_files += 1 + self.removed_pos_deletes += data_file.record_count + elif data_file.content == DataFileContent.EQUALITY_DELETES: + self.removed_delete_files += 1 + self.removed_eq_delete_files += 1 + self.removed_eq_deletes += data_file.record_count + else: + raise ValueError(f"Unknown data file content: {data_file.content}") + + def to_dict(self) -> Dict[str, str]: + properties: Dict[str, str] = {} + set_when_positive(properties, self.added_file_size, ADDED_FILE_SIZE) + set_when_positive(properties, self.removed_file_size, REMOVED_FILE_SIZE) + set_when_positive(properties, self.added_data_files, ADDED_DATA_FILES) + set_when_positive(properties, self.removed_data_files, DELETED_DATA_FILES) + set_when_positive(properties, self.added_eq_delete_files, ADDED_EQUALITY_DELETE_FILES) + set_when_positive(properties, self.removed_eq_delete_files, REMOVED_EQUALITY_DELETE_FILES) + set_when_positive(properties, self.added_pos_delete_files, ADDED_POSITION_DELETE_FILES) + set_when_positive(properties, self.removed_pos_delete_files, REMOVED_POSITION_DELETE_FILES) + set_when_positive(properties, self.added_delete_files, ADDED_DELETE_FILES) + set_when_positive(properties, self.removed_delete_files, REMOVED_DELETE_FILES) + set_when_positive(properties, self.added_records, ADDED_RECORDS) + set_when_positive(properties, self.deleted_records, DELETED_RECORDS) + set_when_positive(properties, self.added_pos_deletes, ADDED_POSITION_DELETES) + set_when_positive(properties, self.removed_pos_deletes, REMOVED_POSITION_DELETES) + set_when_positive(properties, self.added_eq_deletes, ADDED_EQUALITY_DELETES) + set_when_positive(properties, self.removed_eq_deletes, REMOVED_EQUALITY_DELETES) + return properties + + class Summary(IcebergBaseModel, Mapping[str, str]): """A class that stores the summary information for a Snapshot. @@ -172,100 +260,53 @@ class SnapshotLogEntry(IcebergBaseModel): class SnapshotSummaryCollector: - added_file_size: int - removed_file_size: int - added_data_files: int - removed_data_files: int - added_eq_delete_files: int - removed_eq_delete_files: int - added_pos_delete_files: int - removed_pos_delete_files: int - added_delete_files: int - removed_delete_files: int - added_records: int - deleted_records: int - added_pos_deletes: int - removed_pos_deletes: int - added_eq_deletes: int - removed_eq_deletes: int + metrics: UpdateMetrics + partition_metrics: DefaultDict[str, UpdateMetrics] + max_changed_partitions_for_summaries: int def __init__(self) -> None: - self.added_file_size = 0 - self.removed_file_size = 0 - self.added_data_files = 0 - self.removed_data_files = 0 - self.added_eq_delete_files = 0 - self.removed_eq_delete_files = 0 - self.added_pos_delete_files = 0 - self.removed_pos_delete_files = 0 - self.added_delete_files = 0 - self.removed_delete_files = 0 - self.added_records = 0 - self.deleted_records = 0 - self.added_pos_deletes = 0 - self.removed_pos_deletes = 0 - self.added_eq_deletes = 0 - self.removed_eq_deletes = 0 - - def add_file(self, data_file: DataFile) -> None: - self.added_file_size += data_file.file_size_in_bytes - - if data_file.content == DataFileContent.DATA: - self.added_data_files += 1 - self.added_records += data_file.record_count - elif data_file.content == DataFileContent.POSITION_DELETES: - self.added_delete_files += 1 - self.added_pos_delete_files += 1 - self.added_pos_deletes += data_file.record_count - elif data_file.content == DataFileContent.EQUALITY_DELETES: - self.added_delete_files += 1 - self.added_eq_delete_files += 1 - self.added_eq_deletes += data_file.record_count - else: - raise ValueError(f"Unknown data file content: {data_file.content}") - - def remove_file(self, data_file: DataFile) -> None: - self.removed_file_size += data_file.file_size_in_bytes - - if data_file.content == DataFileContent.DATA: - self.removed_data_files += 1 - self.deleted_records += data_file.record_count - elif data_file.content == DataFileContent.POSITION_DELETES: - self.removed_delete_files += 1 - self.removed_pos_delete_files += 1 - self.removed_pos_deletes += data_file.record_count - elif data_file.content == DataFileContent.EQUALITY_DELETES: - self.removed_delete_files += 1 - self.removed_eq_delete_files += 1 - self.removed_eq_deletes += data_file.record_count + self.metrics = UpdateMetrics() + self.partition_metrics = defaultdict(UpdateMetrics) + self.max_changed_partitions_for_summaries = 0 + + def set_partition_summary_limit(self, limit: int) -> None: + self.max_changed_partitions_for_summaries = limit + + def add_file(self, data_file: DataFile, schema: Schema, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC) -> None: + self.metrics.add_file(data_file) + if len(data_file.partition.record_fields()) != 0: + self.update_partition_metrics(partition_spec=partition_spec, file=data_file, is_add_file=True, schema=schema) + + def remove_file( + self, data_file: DataFile, schema: Schema, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC + ) -> None: + self.metrics.remove_file(data_file) + if len(data_file.partition.record_fields()) != 0: + self.update_partition_metrics(partition_spec=partition_spec, file=data_file, is_add_file=False, schema=schema) + + def update_partition_metrics(self, partition_spec: PartitionSpec, file: DataFile, is_add_file: bool, schema: Schema) -> None: + partition_path = partition_spec.partition_to_path(file.partition, schema) + partition_metrics: UpdateMetrics = self.partition_metrics[partition_path] + + if is_add_file: + partition_metrics.add_file(file) else: - raise ValueError(f"Unknown data file content: {data_file.content}") + partition_metrics.remove_file(file) def build(self) -> Dict[str, str]: - def set_when_positive(properties: Dict[str, str], num: int, property_name: str) -> None: - if num > 0: - properties[property_name] = str(num) - - properties: Dict[str, str] = {} - set_when_positive(properties, self.added_file_size, ADDED_FILE_SIZE) - set_when_positive(properties, self.removed_file_size, REMOVED_FILE_SIZE) - set_when_positive(properties, self.added_data_files, ADDED_DATA_FILES) - set_when_positive(properties, self.removed_data_files, DELETED_DATA_FILES) - set_when_positive(properties, self.added_eq_delete_files, ADDED_EQUALITY_DELETE_FILES) - set_when_positive(properties, self.removed_eq_delete_files, REMOVED_EQUALITY_DELETE_FILES) - set_when_positive(properties, self.added_pos_delete_files, ADDED_POSITION_DELETE_FILES) - set_when_positive(properties, self.removed_pos_delete_files, REMOVED_POSITION_DELETE_FILES) - set_when_positive(properties, self.added_delete_files, ADDED_DELETE_FILES) - set_when_positive(properties, self.removed_delete_files, REMOVED_DELETE_FILES) - set_when_positive(properties, self.added_records, ADDED_RECORDS) - set_when_positive(properties, self.deleted_records, DELETED_RECORDS) - set_when_positive(properties, self.added_pos_deletes, ADDED_POSITION_DELETES) - set_when_positive(properties, self.removed_pos_deletes, REMOVED_POSITION_DELETES) - set_when_positive(properties, self.added_eq_deletes, ADDED_EQUALITY_DELETES) - set_when_positive(properties, self.removed_eq_deletes, REMOVED_EQUALITY_DELETES) + properties = self.metrics.to_dict() + changed_partitions_size = len(self.partition_metrics) + set_when_positive(properties, changed_partitions_size, CHANGED_PARTITION_COUNT_PROP) + if changed_partitions_size <= self.max_changed_partitions_for_summaries: + for partition_path, update_metrics_partition in self.partition_metrics.items(): + if (summary := self._partition_summary(update_metrics_partition)) and len(summary) != 0: + properties[CHANGED_PARTITION_PREFIX + partition_path] = summary return properties + def _partition_summary(self, update_metrics: UpdateMetrics) -> str: + return ",".join([f"{prop}={val}" for prop, val in update_metrics.to_dict().items()]) + def _truncate_table_summary(summary: Summary, previous_summary: Mapping[str, str]) -> Summary: for prop in { @@ -366,3 +407,8 @@ def _update_totals(total_property: str, added_property: str, removed_property: s ) return summary + + +def set_when_positive(properties: Dict[str, str], num: int, property_name: str) -> None: + if num > 0: + properties[property_name] = str(num) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index e678a77e69..6dcae59e49 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -35,6 +35,7 @@ BoundLessThan, BoundLessThanOrEqual, BoundLiteralPredicate, + BoundNotEqualTo, BoundNotIn, BoundNotStartsWith, BoundPredicate, @@ -43,8 +44,11 @@ BoundTerm, BoundUnaryPredicate, EqualTo, + GreaterThan, GreaterThanOrEqual, + LessThan, LessThanOrEqual, + NotEqualTo, NotStartsWith, Reference, StartsWith, @@ -144,6 +148,9 @@ def result_type(self, source: IcebergType) -> IcebergType: ... @abstractmethod def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: ... + @abstractmethod + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: ... + @property def preserves_order(self) -> bool: return False @@ -216,6 +223,21 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica # For example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. return None + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: + transformer = self.transform(pred.term.ref().field.field_type) + + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + elif isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundNotEqualTo): + return pred.as_unbound(Reference(name), _transform_literal(transformer, pred.literal)) + elif isinstance(pred, BoundNotIn): + return pred.as_unbound(Reference(name), {_transform_literal(transformer, literal) for literal in pred.literals}) + else: + # no strict projection for comparison or equality + return None + def can_transform(self, source: IcebergType) -> bool: return isinstance( source, @@ -306,6 +328,19 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica else: return None + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: + transformer = self.transform(pred.term.ref().field.field_type) + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + elif isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundLiteralPredicate): + return _truncate_number_strict(name, pred, transformer) + elif isinstance(pred, BoundNotIn): + return _set_apply_transform(name, pred, transformer) + else: + return None + @property def dedup_name(self) -> str: return "time" @@ -516,10 +551,20 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica return pred.as_unbound(Reference(name)) elif isinstance(pred, BoundLiteralPredicate): return pred.as_unbound(Reference(name), pred.literal) - elif isinstance(pred, (BoundIn, BoundNotIn)): + elif isinstance(pred, BoundSetPredicate): + return pred.as_unbound(Reference(name), pred.literals) + else: + return None + + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: + if isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundLiteralPredicate): + return pred.as_unbound(Reference(name), pred.literal) + elif isinstance(pred, BoundSetPredicate): return pred.as_unbound(Reference(name), pred.literals) else: - raise ValueError(f"Could not project: {pred}") + return None @property def preserves_order(self) -> bool: @@ -590,6 +635,47 @@ def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredica return _truncate_array(name, pred, self.transform(field_type)) return None + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: + field_type = pred.term.ref().field.field_type + + if isinstance(pred.term, BoundTransform): + return _project_transform_predicate(self, name, pred) + + if isinstance(field_type, (IntegerType, LongType, DecimalType)): + if isinstance(pred, BoundUnaryPredicate): + return pred.as_unbound(Reference(name)) + elif isinstance(pred, BoundLiteralPredicate): + return _truncate_number_strict(name, pred, self.transform(field_type)) + elif isinstance(pred, BoundNotIn): + return _set_apply_transform(name, pred, self.transform(field_type)) + else: + return None + + if isinstance(pred, BoundLiteralPredicate): + if isinstance(pred, BoundStartsWith): + literal_width = len(pred.literal.value) + if literal_width < self.width: + return pred.as_unbound(name, pred.literal.value) + elif literal_width == self.width: + return EqualTo(name, pred.literal.value) + else: + return None + elif isinstance(pred, BoundNotStartsWith): + literal_width = len(pred.literal.value) + if literal_width < self.width: + return pred.as_unbound(name, pred.literal.value) + elif literal_width == self.width: + return NotEqualTo(name, pred.literal.value) + else: + return pred.as_unbound(name, self.transform(field_type)(pred.literal.value)) + else: + # ProjectionUtil.truncateArrayStrict(name, pred, this); + return _truncate_array_strict(name, pred, self.transform(field_type)) + elif isinstance(pred, BoundNotIn): + return _set_apply_transform(name, pred, self.transform(field_type)) + else: + return None + @property def width(self) -> int: return self._width @@ -714,6 +800,9 @@ def result_type(self, source: IcebergType) -> StringType: def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: return None + def strict_project(self, name: str, pred: BoundPredicate[Any]) -> Optional[UnboundPredicate[Any]]: + return None + def __repr__(self) -> str: """Return the string representation of the UnknownTransform class.""" return f"UnknownTransform(transform={repr(self._transform)})" @@ -736,6 +825,9 @@ def result_type(self, source: IcebergType) -> IcebergType: def project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: return None + def strict_project(self, name: str, pred: BoundPredicate[L]) -> Optional[UnboundPredicate[Any]]: + return None + def to_human_string(self, _: IcebergType, value: Optional[S]) -> str: return "null" @@ -766,6 +858,47 @@ def _truncate_number( return None +def _truncate_number_strict( + name: str, pred: BoundLiteralPredicate[L], transform: Callable[[Optional[L]], Optional[L]] +) -> Optional[UnboundPredicate[Any]]: + boundary = pred.literal + + if not isinstance(boundary, (LongLiteral, DecimalLiteral, DateLiteral, TimestampLiteral)): + raise ValueError(f"Expected a numeric literal, got: {type(boundary)}") + + if isinstance(pred, BoundLessThan): + return LessThan(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundLessThanOrEqual): + return LessThan(Reference(name), _transform_literal(transform, boundary.increment())) # type: ignore + elif isinstance(pred, BoundGreaterThan): + return GreaterThan(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundGreaterThanOrEqual): + return GreaterThan(Reference(name), _transform_literal(transform, boundary.decrement())) # type: ignore + elif isinstance(pred, BoundNotEqualTo): + return EqualTo(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, BoundEqualTo): + # there is no predicate that guarantees equality because adjacent longs transform to the + # same value + return None + else: + return None + + +def _truncate_array_strict( + name: str, pred: BoundLiteralPredicate[L], transform: Callable[[Optional[L]], Optional[L]] +) -> Optional[UnboundPredicate[Any]]: + boundary = pred.literal + + if isinstance(pred, (BoundLessThan, BoundLessThanOrEqual)): + return LessThan(Reference(name), _transform_literal(transform, boundary)) + elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): + return GreaterThan(Reference(name), _transform_literal(transform, boundary)) + if isinstance(pred, BoundNotEqualTo): + return NotEqualTo(Reference(name), _transform_literal(transform, boundary)) + else: + return None + + def _truncate_array( name: str, pred: BoundLiteralPredicate[L], transform: Callable[[Optional[L]], Optional[L]] ) -> Optional[UnboundPredicate[Any]]: @@ -808,7 +941,8 @@ def _remove_transform(partition_name: str, pred: BoundPredicate[L]) -> UnboundPr def _set_apply_transform(name: str, pred: BoundSetPredicate[L], transform: Callable[[L], L]) -> UnboundPredicate[Any]: literals = pred.literals if isinstance(pred, BoundSetPredicate): - return pred.as_unbound(Reference(name), {_transform_literal(transform, literal) for literal in literals}) + transformed_literals = {_transform_literal(transform, literal) for literal in literals} + return pred.as_unbound(Reference(name=name), literals=transformed_literals) else: raise ValueError(f"Unknown BoundSetPredicate: {pred}") diff --git a/pyiceberg/utils/schema_conversion.py b/pyiceberg/utils/schema_conversion.py index 2cceecc639..3cba428dd9 100644 --- a/pyiceberg/utils/schema_conversion.py +++ b/pyiceberg/utils/schema_conversion.py @@ -527,7 +527,9 @@ def field(self, field: NestedField, field_result: AvroType) -> AvroType: "type": field_result if field.required else ["null", field_result], } - if field.optional: + if field.write_default is not None: + result["default"] = field.write_default # type: ignore + elif field.optional: result["default"] = None if field.doc is not None: diff --git a/pyproject.toml b/pyproject.toml index 52c60d9482..de983f8c9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,14 +47,14 @@ include = [ [tool.poetry.dependencies] python = "^3.8" -mmhash3 = ">=3.0.0,<4.0.0" +mmh3 = ">=4.0.0,<5.0.0" requests = ">=2.20.0,<3.0.0" click = ">=7.1.1,<9.0.0" rich = ">=10.11.0,<14.0.0" strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4. pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug sortedcontainers = "2.4.0" -fsspec = ">=2023.1.0,<2024.1.0" +fsspec = ">=2023.1.0,<2025.1.0" pyparsing = ">=3.1.0,<4.0.0" zstandard = ">=0.13.0,<1.0.0" tenacity = ">=8.2.3,<9.0.0" @@ -73,7 +73,7 @@ psycopg2-binary = { version = ">=2.9.6", optional = true } sqlalchemy = { version = "^2.0.18", optional = true } getdaft = { version = ">=0.2.12", optional = true } -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "7.4.4" pytest-checkdocs = "2.10.1" pytest-lazy-fixture = "0.6.3" @@ -86,11 +86,480 @@ typing-extensions = "4.9.0" pytest-mock = "3.12.0" pyspark = "3.5.0" cython = "3.0.8" +deptry = "^0.14.0" [[tool.mypy.overrides]] module = "pytest_mock.*" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "pyarrow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "snappy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "zstandard.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic_core.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pytest.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fastavro.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mmh3.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "hive_metastore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "thrift.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "requests_mock.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "click.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "rich.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fsspec.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "s3fs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "azure.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "adlfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "gcsfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "packaging.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "boto3" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "botocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mypy_boto3_glue.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "moto" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiobotocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiohttp.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "duckdb.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "ray.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "daft.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyparsing.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyspark.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "strictyaml.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sortedcontainers.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "numpy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sqlalchemy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "Cython.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "setuptools.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tenacity.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyarrow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "snappy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "zstandard.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic_core.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pytest.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fastavro.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mmh3.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "hive_metastore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "thrift.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "requests_mock.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "click.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "rich.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fsspec.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "s3fs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "azure.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "adlfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "gcsfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "packaging.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "boto3" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "botocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mypy_boto3_glue.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "moto" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiobotocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiohttp.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "duckdb.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "ray.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "daft.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyparsing.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyspark.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "strictyaml.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sortedcontainers.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "numpy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sqlalchemy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "Cython.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "setuptools.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tenacity.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyarrow.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pandas.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "snappy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "zstandard.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pydantic_core.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pytest.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fastavro.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mmh3.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "hive_metastore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "thrift.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "requests_mock.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "click.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "rich.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "fsspec.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "s3fs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "azure.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "adlfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "gcsfs.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "packaging.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "boto3" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "botocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "mypy_boto3_glue.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "moto" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiobotocore.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "aiohttp.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "duckdb.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "ray.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "daft.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyparsing.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "pyspark.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "strictyaml.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sortedcontainers.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "numpy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "sqlalchemy.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "Cython.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "setuptools.*" +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tenacity.*" +ignore_missing_imports = true + [tool.poetry.scripts] pyiceberg = "pyiceberg.cli.console:run" @@ -149,6 +618,7 @@ warn_unreachable = true warn_unused_ignores = true disallow_any_generics = true disallow_untyped_defs = true +implicit_reexport = false [[tool.mypy.overrides]] module = "pyarrow.*" diff --git a/tests/avro/test_file.py b/tests/avro/test_file.py index 74458fd923..0809f56fea 100644 --- a/tests/avro/test_file.py +++ b/tests/avro/test_file.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import inspect +from copy import copy from datetime import date, datetime, time from enum import Enum from tempfile import TemporaryDirectory @@ -26,7 +27,7 @@ from fastavro import reader, writer import pyiceberg.avro.file as avro -from pyiceberg.avro.codecs import DeflateCodec +from pyiceberg.avro.codecs.deflate import DeflateCodec from pyiceberg.avro.file import META_SCHEMA, AvroFileHeader from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import ( @@ -111,7 +112,7 @@ def todict(obj: Any) -> Any: return obj.value elif hasattr(obj, "__iter__") and not isinstance(obj, str) and not isinstance(obj, bytes): return [todict(v) for v in obj] - elif hasattr(obj, "__dict__"): + elif isinstance(obj, Record): return {key: todict(value) for key, value in inspect.getmembers(obj) if not callable(value) and not key.startswith("_")} else: return obj @@ -258,8 +259,6 @@ def test_write_manifest_entry_with_fastavro_read_with_iceberg(format_version: in sort_order_id=4, spec_id=3, ) - if format_version == 1: - data_file.block_size_in_bytes = DEFAULT_BLOCK_SIZE entry = ManifestEntry( status=ManifestEntryStatus.ADDED, @@ -277,16 +276,44 @@ def test_write_manifest_entry_with_fastavro_read_with_iceberg(format_version: in with open(tmp_avro_file, "wb") as out: writer(out, schema, [todict(entry)]) + # Read as V2 with avro.AvroFile[ManifestEntry]( - PyArrowFileIO().new_input(tmp_avro_file), - MANIFEST_ENTRY_SCHEMAS[format_version], - {-1: ManifestEntry, 2: DataFile}, + input_file=PyArrowFileIO().new_input(tmp_avro_file), + read_schema=MANIFEST_ENTRY_SCHEMAS[2], + read_types={-1: ManifestEntry, 2: DataFile}, ) as avro_reader: it = iter(avro_reader) avro_entry = next(it) assert entry == avro_entry + # Read as the original version + with avro.AvroFile[ManifestEntry]( + input_file=PyArrowFileIO().new_input(tmp_avro_file), + read_schema=MANIFEST_ENTRY_SCHEMAS[format_version], + read_types={-1: ManifestEntry, 2: DataFile}, + ) as avro_reader: + it = iter(avro_reader) + avro_entry = next(it) + + if format_version == 1: + v1_datafile = copy(data_file) + # Not part of V1 + v1_datafile.equality_ids = None + + assert avro_entry == ManifestEntry( + status=ManifestEntryStatus.ADDED, + snapshot_id=8638475580105682862, + # Not part of v1 + data_sequence_number=None, + file_sequence_number=None, + data_file=v1_datafile, + ) + elif format_version == 2: + assert entry == avro_entry + else: + raise ValueError(f"Unsupported version: {format_version}") + @pytest.mark.parametrize("is_required", [True, False]) def test_all_primitive_types(is_required: bool) -> None: diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py index c08ca235fb..07d41491fa 100644 --- a/tests/avro/test_resolver.py +++ b/tests/avro/test_resolver.py @@ -32,7 +32,7 @@ StringReader, StructReader, ) -from pyiceberg.avro.resolver import ResolveError, resolve_reader, resolve_writer +from pyiceberg.avro.resolver import resolve_reader, resolve_writer from pyiceberg.avro.writer import ( BinaryWriter, DefaultWriter, @@ -44,6 +44,7 @@ StringWriter, StructWriter, ) +from pyiceberg.exceptions import ResolveError from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import MANIFEST_ENTRY_SCHEMAS from pyiceberg.schema import Schema diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py index 1f0060780e..5f78eb3bc4 100644 --- a/tests/catalog/test_base.py +++ b/tests/catalog/test_base.py @@ -16,6 +16,9 @@ # under the License. # pylint:disable=redefined-outer-name + +import uuid +from pathlib import PosixPath from typing import ( Dict, List, @@ -40,7 +43,7 @@ NoSuchTableError, TableAlreadyExistsError, ) -from pyiceberg.io import load_file_io +from pyiceberg.io import WAREHOUSE, load_file_io from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import ( @@ -53,15 +56,21 @@ TableIdentifier, update_table_metadata, ) -from pyiceberg.table.metadata import TableMetadataV1 +from pyiceberg.table.metadata import new_table_metadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import EMPTY_DICT, Identifier, Properties from pyiceberg.types import IntegerType, LongType, NestedField +DEFAULT_WAREHOUSE_LOCATION = "file:///tmp/warehouse" + class InMemoryCatalog(Catalog): - """An in-memory catalog implementation for testing purposes.""" + """ + An in-memory catalog implementation that uses in-memory data-structures to store the namespaces and tables. + + This is useful for test, demo, and playground but not in production as data is not persisted. + """ __tables: Dict[Identifier, Table] __namespaces: Dict[Identifier, Properties] @@ -70,6 +79,7 @@ def __init__(self, name: str, **properties: str) -> None: super().__init__(name, **properties) self.__tables = {} self.__namespaces = {} + self._warehouse_location = properties.get(WAREHOUSE, DEFAULT_WAREHOUSE_LOCATION) def create_table( self, @@ -79,6 +89,7 @@ def create_table( partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC, sort_order: SortOrder = UNSORTED_SORT_ORDER, properties: Properties = EMPTY_DICT, + table_uuid: Optional[uuid.UUID] = None, ) -> Table: schema: Schema = self._convert_schema_if_needed(schema) # type: ignore @@ -91,24 +102,26 @@ def create_table( if namespace not in self.__namespaces: self.__namespaces[namespace] = {} - new_location = location or f's3://warehouse/{"/".join(identifier)}/data' - metadata = TableMetadataV1(**{ - "format-version": 1, - "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", - "location": new_location, - "last-updated-ms": 1602638573874, - "last-column-id": schema.highest_field_id, - "schema": schema.model_dump(), - "partition-spec": partition_spec.model_dump()["fields"], - "properties": properties, - "current-snapshot-id": -1, - "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], - }) + if not location: + location = f'{self._warehouse_location}/{"/".join(identifier)}' + + metadata_location = self._get_metadata_location(location=location) + metadata = new_table_metadata( + schema=schema, + partition_spec=partition_spec, + sort_order=sort_order, + location=location, + properties=properties, + table_uuid=table_uuid, + ) + io = load_file_io({**self.properties, **properties}, location=location) + self._write_metadata(metadata, io, metadata_location) + table = Table( identifier=identifier, metadata=metadata, - metadata_location=f's3://warehouse/{"/".join(identifier)}/metadata/metadata.json', - io=load_file_io(), + metadata_location=metadata_location, + io=io, catalog=self, ) self.__tables[identifier] = table @@ -118,14 +131,29 @@ def register_table(self, identifier: Union[str, Identifier], metadata_location: raise NotImplementedError def _commit_table(self, table_request: CommitTableRequest) -> CommitTableResponse: - identifier = tuple(table_request.identifier.namespace.root) + (table_request.identifier.name,) - table = self.__tables[identifier] - table.metadata = update_table_metadata(base_metadata=table.metadata, updates=table_request.updates) - - return CommitTableResponse( - metadata=table.metadata.model_dump(), - metadata_location=table.location(), + identifier_tuple = self.identifier_to_tuple_without_catalog( + tuple(table_request.identifier.namespace.root + [table_request.identifier.name]) ) + current_table = self.load_table(identifier_tuple) + base_metadata = current_table.metadata + + for requirement in table_request.requirements: + requirement.validate(base_metadata) + + updated_metadata = update_table_metadata(base_metadata, table_request.updates) + if updated_metadata == base_metadata: + # no changes, do nothing + return CommitTableResponse(metadata=base_metadata, metadata_location=current_table.metadata_location) + + # write new metadata + new_metadata_version = self._parse_metadata_version(current_table.metadata_location) + 1 + new_metadata_location = self._get_metadata_location(current_table.metadata.location, new_metadata_version) + self._write_metadata(updated_metadata, current_table.io, new_metadata_location) + + # update table state + current_table.metadata = updated_metadata + + return CommitTableResponse(metadata=updated_metadata, metadata_location=new_metadata_location) def load_table(self, identifier: Union[str, Identifier]) -> Table: identifier = self.identifier_to_tuple_without_catalog(identifier) @@ -160,7 +188,7 @@ def rename_table(self, from_identifier: Union[str, Identifier], to_identifier: U identifier=to_identifier, metadata=table.metadata, metadata_location=table.metadata_location, - io=load_file_io(), + io=self._load_file_io(properties=table.metadata.properties, location=table.metadata_location), catalog=self, ) return self.__tables[to_identifier] @@ -232,8 +260,8 @@ def update_namespace_properties( @pytest.fixture -def catalog() -> InMemoryCatalog: - return InMemoryCatalog("test.in.memory.catalog", **{"test.key": "test.value"}) +def catalog(tmp_path: PosixPath) -> InMemoryCatalog: + return InMemoryCatalog("test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"}) TEST_TABLE_IDENTIFIER = ("com", "organization", "department", "my_table") @@ -244,7 +272,6 @@ def catalog() -> InMemoryCatalog: NestedField(2, "y", LongType(), doc="comment"), NestedField(3, "z", LongType()), ) -TEST_TABLE_LOCATION = "protocol://some/location" TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000)) TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"} NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)" @@ -261,7 +288,6 @@ def given_catalog_has_a_table( return catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=properties or TEST_TABLE_PROPERTIES, ) @@ -307,13 +333,25 @@ def test_create_table(catalog: InMemoryCatalog) -> None: table = catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table +def test_create_table_location_override(catalog: InMemoryCatalog) -> None: + new_location = f"{catalog._warehouse_location}/new_location" + table = catalog.create_table( + identifier=TEST_TABLE_IDENTIFIER, + schema=TEST_TABLE_SCHEMA, + location=new_location, + partition_spec=TEST_TABLE_PARTITION_SPEC, + properties=TEST_TABLE_PROPERTIES, + ) + assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table + assert table.location() == new_location + + @pytest.mark.parametrize( "schema,expected", [ @@ -335,8 +373,6 @@ def test_create_table_pyarrow_schema(catalog: InMemoryCatalog, pyarrow_schema_si table = catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=pyarrow_schema_simple_without_ids, - location=TEST_TABLE_LOCATION, - partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) assert catalog.load_table(TEST_TABLE_IDENTIFIER) == table @@ -377,6 +413,17 @@ def test_table_raises_error_on_table_not_found(catalog: InMemoryCatalog) -> None catalog.load_table(TEST_TABLE_IDENTIFIER) +def test_table_exists(catalog: InMemoryCatalog) -> None: + # Given + given_catalog_has_a_table(catalog) + # Then + assert catalog.table_exists(TEST_TABLE_IDENTIFIER) + + +def test_table_exists_on_table_not_found(catalog: InMemoryCatalog) -> None: + assert not catalog.table_exists(TEST_TABLE_IDENTIFIER) + + def test_drop_table(catalog: InMemoryCatalog) -> None: # Given given_catalog_has_a_table(catalog) @@ -662,7 +709,7 @@ def test_add_column_with_statement(catalog: InMemoryCatalog) -> None: def test_catalog_repr(catalog: InMemoryCatalog) -> None: s = repr(catalog) - assert s == "test.in.memory.catalog ()" + assert s == "test.in_memory.catalog ()" def test_table_properties_int_value(catalog: InMemoryCatalog) -> None: diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index 6d44d92724..d4ed085c51 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -32,6 +32,7 @@ NoSuchTableError, TableAlreadyExistsError, ) +from pyiceberg.io.pyarrow import schema_to_pyarrow from pyiceberg.schema import Schema from pyiceberg.types import IntegerType from tests.conftest import BUCKET_NAME, TABLE_METADATA_LOCATION_REGEX @@ -692,3 +693,68 @@ def test_commit_table_properties( updated_table_metadata = table.metadata assert test_catalog._parse_metadata_version(table.metadata_location) == 1 assert updated_table_metadata.properties == {"test_a": "test_aa", "test_c": "test_c"} + + +@mock_aws +def test_commit_append_table_snapshot_properties( + _bucket_initialize: None, moto_endpoint_url: str, table_schema_simple: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "glue" + identifier = (database_name, table_name) + test_catalog = GlueCatalog(catalog_name, **{"s3.endpoint": moto_endpoint_url, "warehouse": f"s3://{BUCKET_NAME}"}) + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier=identifier, schema=table_schema_simple) + + assert test_catalog._parse_metadata_version(table.metadata_location) == 0 + + table.append( + pa.Table.from_pylist( + [{"foo": "foo_val", "bar": 1, "baz": False}], + schema=schema_to_pyarrow(table_schema_simple), + ), + snapshot_properties={"snapshot_prop_a": "test_prop_a"}, + ) + + updated_table_metadata = table.metadata + summary = updated_table_metadata.snapshots[-1].summary + assert test_catalog._parse_metadata_version(table.metadata_location) == 1 + assert summary is not None + assert summary["snapshot_prop_a"] == "test_prop_a" + + +@mock_aws +def test_commit_overwrite_table_snapshot_properties( + _bucket_initialize: None, moto_endpoint_url: str, table_schema_simple: Schema, database_name: str, table_name: str +) -> None: + catalog_name = "glue" + identifier = (database_name, table_name) + test_catalog = GlueCatalog(catalog_name, **{"s3.endpoint": moto_endpoint_url, "warehouse": f"s3://{BUCKET_NAME}"}) + test_catalog.create_namespace(namespace=database_name) + table = test_catalog.create_table(identifier=identifier, schema=table_schema_simple) + + assert test_catalog._parse_metadata_version(table.metadata_location) == 0 + + table.append( + pa.Table.from_pylist( + [{"foo": "foo_val", "bar": 1, "baz": False}], + schema=schema_to_pyarrow(table_schema_simple), + ), + snapshot_properties={"snapshot_prop_a": "test_prop_a"}, + ) + + assert test_catalog._parse_metadata_version(table.metadata_location) == 1 + + table.overwrite( + pa.Table.from_pylist( + [{"foo": "foo_val", "bar": 2, "baz": True}], + schema=schema_to_pyarrow(table_schema_simple), + ), + snapshot_properties={"snapshot_prop_b": "test_prop_b"}, + ) + + updated_table_metadata = table.metadata + summary = updated_table_metadata.snapshots[-1].summary + assert test_catalog._parse_metadata_version(table.metadata_location) == 2 + assert summary is not None + assert summary["snapshot_prop_a"] is None + assert summary["snapshot_prop_b"] == "test_prop_b" diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 850e5f0180..4956fffe6c 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -644,6 +644,26 @@ def test_load_table_404(rest_mock: Mocker) -> None: assert "Table does not exist" in str(e.value) +def test_table_exist_200(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + status_code=200, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + assert catalog.table_exists(("fokko", "table")) + + +def test_table_exist_500(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + status_code=500, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + assert not catalog.table_exists(("fokko", "table")) + + def test_drop_table_404(rest_mock: Mocker) -> None: rest_mock.delete( f"{TEST_URI}v1/namespaces/fokko/tables/does_not_exists", diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index 3a77f8678a..b20f617e32 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -193,6 +193,39 @@ def test_create_table_with_pyarrow_schema( catalog.drop_table(random_identifier) +@pytest.mark.parametrize( + 'catalog', + [ + lazy_fixture('catalog_memory'), + # lazy_fixture('catalog_sqlite'), + ], +) +def test_write_pyarrow_schema(catalog: SqlCatalog, random_identifier: Identifier) -> None: + import pyarrow as pa + + pyarrow_table = pa.Table.from_arrays( + [ + pa.array([None, "A", "B", "C"]), # 'foo' column + pa.array([1, 2, 3, 4]), # 'bar' column + pa.array([True, None, False, True]), # 'baz' column + pa.array([None, "A", "B", "C"]), # 'large' column + ], + schema=pa.schema([ + pa.field('foo', pa.string(), nullable=True), + pa.field('bar', pa.int32(), nullable=False), + pa.field('baz', pa.bool_(), nullable=True), + pa.field('large', pa.large_string(), nullable=True), + ]), + ) + database_name, _table_name = random_identifier + catalog.create_namespace(database_name) + table = catalog.create_table(random_identifier, pyarrow_table.schema) + print(pyarrow_table.schema) + print(table.schema().as_struct()) + print() + table.overwrite(pyarrow_table) + + @pytest.mark.parametrize( 'catalog', [ diff --git a/tests/cli/test_console.py b/tests/cli/test_console.py index d77b290ec6..3c208c0ab1 100644 --- a/tests/cli/test_console.py +++ b/tests/cli/test_console.py @@ -14,13 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import datetime import os +import uuid +from pathlib import PosixPath +from unittest.mock import MagicMock import pytest from click.testing import CliRunner from pytest_mock import MockFixture from pyiceberg.cli.console import run +from pyiceberg.io import WAREHOUSE from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.transforms import IdentityTransform @@ -48,8 +53,10 @@ def env_vars(mocker: MockFixture) -> None: @pytest.fixture(name="catalog") -def fixture_catalog(mocker: MockFixture) -> InMemoryCatalog: - in_memory_catalog = InMemoryCatalog("test.in.memory.catalog", **{"test.key": "test.value"}) +def fixture_catalog(mocker: MockFixture, tmp_path: PosixPath) -> InMemoryCatalog: + in_memory_catalog = InMemoryCatalog( + "test.in_memory.catalog", **{WAREHOUSE: tmp_path.absolute().as_posix(), "test.key": "test.value"} + ) mocker.patch("pyiceberg.cli.console.load_catalog", return_value=in_memory_catalog) return in_memory_catalog @@ -59,6 +66,13 @@ def fixture_namespace_properties() -> Properties: return TEST_NAMESPACE_PROPERTIES.copy() +@pytest.fixture() +def mock_datetime_now(monkeypatch: pytest.MonkeyPatch) -> None: + datetime_mock = MagicMock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime.fromtimestamp(TEST_TIMESTAMP / 1000.0).astimezone() + monkeypatch.setattr(datetime, "datetime", datetime_mock) + + TEST_TABLE_IDENTIFIER = ("default", "my_table") TEST_TABLE_NAMESPACE = "default" TEST_NAMESPACE_PROPERTIES = {"location": "s3://warehouse/database/location"} @@ -68,9 +82,10 @@ def fixture_namespace_properties() -> Properties: NestedField(2, "y", LongType(), doc="comment"), NestedField(3, "z", LongType()), ) -TEST_TABLE_LOCATION = "s3://bucket/test/location" TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000)) TEST_TABLE_PROPERTIES = {"read.split.target.size": "134217728"} +TEST_TABLE_UUID = uuid.UUID("d20125c8-7284-442c-9aea-15fee620737c") +TEST_TIMESTAMP = 1602638573874 MOCK_ENVIRONMENT = {"PYICEBERG_CATALOG__PRODUCTION__URI": "test://doesnotexist"} @@ -88,7 +103,6 @@ def test_list_namespace(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -120,12 +134,13 @@ def test_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> None: assert result.output == "Namespace does not exist: ('doesnotexist',)\n" -def test_describe_table(catalog: InMemoryCatalog) -> None: +@pytest.fixture() +def test_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, + table_uuid=TEST_TABLE_UUID, ) runner = CliRunner() @@ -134,7 +149,7 @@ def test_describe_table(catalog: InMemoryCatalog) -> None: assert ( # Strip the whitespace on the end "\n".join([line.rstrip() for line in result.output.split("\n")]) - == """Table format version 1 + == """Table format version 2 Metadata location s3://warehouse/default/my_table/metadata/metadata.json Table UUID d20125c8-7284-442c-9aea-15fee620737c Last Updated 1602638573874 @@ -167,7 +182,6 @@ def test_schema(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -196,7 +210,6 @@ def test_spec(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -225,8 +238,8 @@ def test_uuid(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, + table_uuid=TEST_TABLE_UUID, ) runner = CliRunner() @@ -248,14 +261,12 @@ def test_location(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) - runner = CliRunner() result = runner.invoke(run, ["location", "default.my_table"]) assert result.exit_code == 0 - assert result.output == """s3://bucket/test/location\n""" + assert result.output == f"""{catalog._warehouse_location}/default/my_table\n""" def test_location_does_not_exists(catalog: InMemoryCatalog) -> None: @@ -271,7 +282,6 @@ def test_drop_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -328,7 +338,6 @@ def test_rename_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -351,7 +360,6 @@ def test_properties_get_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -366,7 +374,6 @@ def test_properties_get_table_specific_property(catalog: InMemoryCatalog) -> Non catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -381,7 +388,6 @@ def test_properties_get_table_specific_property_that_doesnt_exist(catalog: InMem catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -450,7 +456,6 @@ def test_properties_set_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -491,7 +496,6 @@ def test_properties_remove_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -506,7 +510,6 @@ def test_properties_remove_table_property_does_not_exists(catalog: InMemoryCatal catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -538,7 +541,6 @@ def test_json_list_namespace(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -566,12 +568,13 @@ def test_json_describe_namespace_does_not_exists(catalog: InMemoryCatalog) -> No assert result.output == """{"type": "NoSuchNamespaceError", "message": "Namespace does not exist: ('doesnotexist',)"}\n""" -def test_json_describe_table(catalog: InMemoryCatalog) -> None: +@pytest.fixture() +def test_json_describe_table(catalog: InMemoryCatalog, mock_datetime_now: None) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, + table_uuid=TEST_TABLE_UUID, ) runner = CliRunner() @@ -579,7 +582,7 @@ def test_json_describe_table(catalog: InMemoryCatalog) -> None: assert result.exit_code == 0 assert ( result.output - == """{"identifier":["default","my_table"],"metadata_location":"s3://warehouse/default/my_table/metadata/metadata.json","metadata":{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}],"current-schema-id":0,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{},"snapshots":[{"snapshot-id":1925,"timestamp-ms":1602638573822}],"snapshot-log":[],"metadata-log":[],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0,"refs":{},"format-version":1,"schema":{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},"partition-spec":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}}\n""" + == """{"identifier":["default","my_table"],"metadata_location":"s3://warehouse/default/my_table/metadata/metadata.json","metadata":{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}],"current-schema-id":0,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{},"snapshots":[{"snapshot-id":1925,"timestamp-ms":1602638573822}],"snapshot-log":[],"metadata-log":[],"sort-orders":[{"order-id":0,"fields":[]}],"default-sort-order-id":0,"refs":{},"format-version":2,"schema":{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},"partition-spec":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}}\n""" ) @@ -599,7 +602,6 @@ def test_json_schema(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -625,7 +627,6 @@ def test_json_spec(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -648,8 +649,8 @@ def test_json_uuid(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, + table_uuid=TEST_TABLE_UUID, ) runner = CliRunner() @@ -671,14 +672,13 @@ def test_json_location(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) runner = CliRunner() result = runner.invoke(run, ["--output=json", "location", "default.my_table"]) assert result.exit_code == 0 - assert result.output == """"s3://bucket/test/location"\n""" + assert result.output == f'"{catalog._warehouse_location}/default/my_table"\n' def test_json_location_does_not_exists(catalog: InMemoryCatalog) -> None: @@ -694,7 +694,6 @@ def test_json_drop_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -735,7 +734,6 @@ def test_json_rename_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, ) @@ -758,7 +756,6 @@ def test_json_properties_get_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -773,7 +770,6 @@ def test_json_properties_get_table_specific_property(catalog: InMemoryCatalog) - catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -788,7 +784,6 @@ def test_json_properties_get_table_specific_property_that_doesnt_exist(catalog: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -862,7 +857,6 @@ def test_json_properties_set_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -908,7 +902,6 @@ def test_json_properties_remove_table(catalog: InMemoryCatalog) -> None: catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) @@ -923,7 +916,6 @@ def test_json_properties_remove_table_property_does_not_exists(catalog: InMemory catalog.create_table( identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, - location=TEST_TABLE_LOCATION, partition_spec=TEST_TABLE_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) diff --git a/tests/conftest.py b/tests/conftest.py index ca13a8c94b..d0f0d5920a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,6 +79,7 @@ NestedField, StringType, StructType, + UUIDType, ) from pyiceberg.utils.datetime import datetime_to_millis @@ -892,7 +893,7 @@ def metadata_location_gz(tmp_path_factory: pytest.TempPathFactory) -> str: "data_file": { "file_path": "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=1/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00002.parquet", "file_format": "PARQUET", - "partition": {"VendorID": 1, "tpep_pickup_datetime": 1925}, + "partition": {"VendorID": 1, "tpep_pickup_datetime": None}, "record_count": 95050, "file_size_in_bytes": 1265950, "block_size_in_bytes": 67108864, @@ -1858,8 +1859,8 @@ def get_s3_path(bucket_name: str, database_name: Optional[str] = None, table_nam @pytest.fixture(name="s3", scope="module") def fixture_s3_client() -> boto3.client: - with mock_aws(): - yield boto3.client("s3") + """Real S3 client for AWS Integration Tests.""" + yield boto3.client("s3") def clean_up(test_catalog: Catalog) -> None: @@ -1928,6 +1929,16 @@ def bound_reference_str() -> BoundReference[str]: return BoundReference(field=NestedField(1, "field", StringType(), required=False), accessor=Accessor(position=0, inner=None)) +@pytest.fixture +def bound_reference_binary() -> BoundReference[str]: + return BoundReference(field=NestedField(1, "field", BinaryType(), required=False), accessor=Accessor(position=0, inner=None)) + + +@pytest.fixture +def bound_reference_uuid() -> BoundReference[str]: + return BoundReference(field=NestedField(1, "field", UUIDType(), required=False), accessor=Accessor(position=0, inner=None)) + + @pytest.fixture(scope="session") def session_catalog() -> Catalog: return load_catalog( diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index 7d97a6d2d2..f8a9a8806d 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -39,7 +39,7 @@ Or, StartsWith, ) -from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator +from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator, _StrictMetricsEvaluator from pyiceberg.manifest import DataFile, FileFormat from pyiceberg.schema import Schema from pyiceberg.types import ( @@ -925,3 +925,530 @@ def test_string_not_starts_with( # should_read = _InclusiveMetricsEvaluator(schema_data_file, NotStartsWith("required", above_max)).eval(data_file_4) # assert should_read, "Should not read: range doesn't match" + + +@pytest.fixture +def strict_data_file_schema() -> Schema: + return Schema( + NestedField(1, "id", IntegerType(), required=True), + NestedField(2, "no_stats", IntegerType(), required=False), + NestedField(3, "required", StringType(), required=True), + NestedField(4, "all_nulls", StringType(), required=False), + NestedField(5, "some_nulls", StringType(), required=False), + NestedField(6, "no_nulls", StringType(), required=False), + NestedField(7, "always_5", IntegerType(), required=False), + NestedField(8, "all_nans", DoubleType(), required=False), + NestedField(9, "some_nans", FloatType(), required=False), + NestedField(10, "no_nans", FloatType(), required=False), + NestedField(11, "all_nulls_double", DoubleType(), required=False), + NestedField(12, "all_nans_v1_stats", FloatType(), required=False), + NestedField(13, "nan_and_null_only", DoubleType(), required=False), + NestedField(14, "no_nan_stats", DoubleType(), required=False), + ) + + +@pytest.fixture +def strict_data_file_1() -> DataFile: + return DataFile( + file_path="file_1.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={ + 4: 50, + 5: 50, + 6: 50, + 8: 50, + 9: 50, + 10: 50, + 11: 50, + 12: 50, + 13: 50, + 14: 50, + }, + null_value_counts={4: 50, 5: 10, 6: 0, 11: 50, 12: 0, 13: 1}, + nan_value_counts={ + 8: 50, + 9: 10, + 10: 0, + }, + lower_bounds={ + 1: to_bytes(IntegerType(), INT_MIN_VALUE), + 7: to_bytes(IntegerType(), 5), + 12: to_bytes(FloatType(), float("nan")), + 13: to_bytes(DoubleType(), float("nan")), + }, + upper_bounds={ + 1: to_bytes(IntegerType(), INT_MAX_VALUE), + 7: to_bytes(IntegerType(), 5), + 12: to_bytes(FloatType(), float("nan")), + 14: to_bytes(DoubleType(), float("nan")), + }, + ) + + +@pytest.fixture +def strict_data_file_2() -> DataFile: + return DataFile( + file_path="file_2.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={ + 4: 50, + 5: 50, + 6: 50, + 8: 50, + }, + null_value_counts={4: 50, 5: 10, 6: 0}, + nan_value_counts=None, + lower_bounds={ + 5: to_bytes(StringType(), "bbb"), + }, + upper_bounds={ + 5: to_bytes(StringType(), "eee"), + }, + ) + + +@pytest.fixture +def strict_data_file_3() -> DataFile: + return DataFile( + file_path="file_3.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + file_size_in_bytes=3, + value_counts={ + 4: 50, + 5: 50, + 6: 50, + }, + null_value_counts={4: 50, 5: 10, 6: 0}, + nan_value_counts=None, + lower_bounds={ + 5: to_bytes(StringType(), "bbb"), + }, + upper_bounds={ + 5: to_bytes(StringType(), "eee"), + }, + ) + + +def test_strict_all_nulls(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNull("all_nulls")).eval(strict_data_file_1) + assert not should_read, "Should not match: no non-null value in all null column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNull("some_nulls")).eval(strict_data_file_1) + assert not should_read, "Should not match: column with some nulls contains a non-null value" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNull("no_nulls")).eval(strict_data_file_1) + assert should_read, "Should match: non-null column contains no null values" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("all_nulls", "a")).eval(strict_data_file_1) + assert should_read, "Should match: notEqual on all nulls column" + + +def test_strict_no_nulls(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNull("all_nulls")).eval(strict_data_file_1) + assert should_read, "Should match: all values are null" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNull("some_nulls")).eval(strict_data_file_1) + assert not should_read, "Should not match: not all values are null" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNull("no_nulls")).eval(strict_data_file_1) + assert not should_read, "Should not match: no values are null" + + +def test_strict_some_nulls(strict_data_file_schema: Schema, strict_data_file_2: DataFile, strict_data_file_3: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThan("some_nulls", "ggg")).eval(strict_data_file_2) + assert not should_read, "Should not match: lessThan on some nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThanOrEqual("some_nulls", "ggg")).eval(strict_data_file_2) + assert not should_read, "Should not match: lessThanOrEqual on some nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThan("some_nulls", "aaa")).eval(strict_data_file_2) + assert not should_read, "Should not match: greaterThan on some nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThanOrEqual("some_nulls", "bbb")).eval( + strict_data_file_2 + ) + assert not should_read, "Should not match: greaterThanOrEqual on some nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("some_nulls", "bbb")).eval(strict_data_file_3) + assert not should_read, "Should not match: equal on some nulls column" + + +def test_strict_is_nan(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("all_nans")).eval(strict_data_file_1) + assert should_read, "Should match: all values are nan" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("some_nans")).eval(strict_data_file_1) + assert not should_read, "Should not match: at least one non-nan value in some nan column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("no_nans")).eval(strict_data_file_1) + assert not should_read, "Should not match: at least one non-nan value in no nan column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("all_nulls_double")).eval(strict_data_file_1) + assert not should_read, "Should not match: at least one non-nan value in all null column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("no_nan_stats")).eval(strict_data_file_1) + assert not should_read, "Should not match: cannot determine without nan stats" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("all_nans_v1_stats")).eval(strict_data_file_1) + assert not should_read, "Should not match: cannot determine without nan stats" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNaN("nan_and_null_only")).eval(strict_data_file_1) + assert not should_read, "Should not match: null values are not nan" + + +def test_strict_not_nan(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("all_nans")).eval(strict_data_file_1) + assert not should_read, "Should not match: all values are nan" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("some_nans")).eval(strict_data_file_1) + assert not should_read, "Should not match: at least one nan value in some nan column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("no_nans")).eval(strict_data_file_1) + assert should_read, "Should match: no value is nan" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("all_nulls_double")).eval(strict_data_file_1) + assert should_read, "Should match: no nan value in all null column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("no_nan_stats")).eval(strict_data_file_1) + assert not should_read, "Should not match: cannot determine without nan stats" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("all_nans_v1_stats")).eval(strict_data_file_1) + assert not should_read, "Should not match: all values are nan" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNaN("nan_and_null_only")).eval(strict_data_file_1) + assert not should_read, "Should not match: null values are not nan" + + +def test_strict_required_column(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotNull("required")).eval(strict_data_file_1) + assert should_read, "Should match: required columns are always non-null" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, IsNull("required")).eval(strict_data_file_1) + assert not should_read, "Should not match: required columns never contain null" + + +def test_strict_missing_column(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + with pytest.raises(ValueError) as exc_info: + _ = _StrictMetricsEvaluator(strict_data_file_schema, NotNull("missing")).eval(strict_data_file_1) + + assert str(exc_info.value) == "Could not find field with name missing, case_sensitive=True" + + +def test_strict_missing_stats(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + no_stats_schema = Schema( + NestedField(2, "no_stats", DoubleType(), required=False), + ) + + no_stats_file = DataFile( + file_path="file_1.parquet", + file_format=FileFormat.PARQUET, + partition={}, + record_count=50, + value_counts=None, + null_value_counts=None, + nan_value_counts=None, + lower_bounds=None, + upper_bounds=None, + ) + + expressions = [ + LessThan("no_stats", 5), + LessThanOrEqual("no_stats", 30), + EqualTo("no_stats", 70), + GreaterThan("no_stats", 78), + GreaterThanOrEqual("no_stats", 90), + NotEqualTo("no_stats", 101), + IsNull("no_stats"), + NotNull("no_stats"), + IsNaN("no_stats"), + NotNaN("no_stats"), + ] + + for expression in expressions: + should_read = _StrictMetricsEvaluator(no_stats_schema, expression).eval(no_stats_file) + assert not should_read, f"Should never match when stats are missing for expr: {expression}" + + +def test_strict_zero_record_file_stats(strict_data_file_schema: Schema) -> None: + zero_record_data_file = DataFile(file_path="file_1.parquet", file_format=FileFormat.PARQUET, partition={}, record_count=0) + + expressions = [ + LessThan("no_stats", 5), + LessThanOrEqual("no_stats", 30), + EqualTo("no_stats", 70), + GreaterThan("no_stats", 78), + GreaterThanOrEqual("no_stats", 90), + NotEqualTo("no_stats", 101), + IsNull("no_stats"), + NotNull("no_stats"), + IsNaN("no_stats"), + NotNaN("no_stats"), + ] + + for expression in expressions: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, expression).eval(zero_record_data_file) + assert should_read, f"Should always match 0-record file: {expression}" + + +def test_strict_not(schema_data_file: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(schema_data_file, Not(LessThan("id", INT_MIN_VALUE - 25))).eval(strict_data_file_1) + assert should_read, "Should not match: not(false)" + + should_read = _StrictMetricsEvaluator(schema_data_file, Not(GreaterThan("id", INT_MIN_VALUE - 25))).eval(strict_data_file_1) + assert not should_read, "Should match: not(true)" + + +def test_strict_and(schema_data_file: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator( + schema_data_file, And(GreaterThan("id", INT_MIN_VALUE - 25), LessThanOrEqual("id", INT_MIN_VALUE)) + ).eval(strict_data_file_1) + assert not should_read, "Should not match: range may not overlap data" + + should_read = _StrictMetricsEvaluator( + schema_data_file, And(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE - 30)) + ).eval(strict_data_file_1) + assert not should_read, "Should not match: range does not overlap data" + + should_read = _StrictMetricsEvaluator( + schema_data_file, And(LessThan("id", INT_MAX_VALUE + 6), GreaterThanOrEqual("id", INT_MIN_VALUE - 30)) + ).eval(strict_data_file_1) + assert should_read, "Should match: range includes all data" + + +def test_strict_or(schema_data_file: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator( + schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE + 1)) + ).eval(strict_data_file_1) + assert not should_read, "Should not match: no matching values" + + should_read = _StrictMetricsEvaluator( + schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MAX_VALUE - 19)) + ).eval(strict_data_file_1) + assert not should_read, "Should not match: some values do not match" + + should_read = _StrictMetricsEvaluator( + schema_data_file, Or(LessThan("id", INT_MIN_VALUE - 25), GreaterThanOrEqual("id", INT_MIN_VALUE)) + ).eval(strict_data_file_1) + assert should_read, "Should match: all values match >= 30" + + +def test_strict_integer_lt(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThan("id", INT_MIN_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: always false" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThan("id", INT_MIN_VALUE + 1)).eval(strict_data_file_1) + assert not should_read, "Should not match: 32 and greater not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThan("id", INT_MAX_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: 79 not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThan("id", INT_MAX_VALUE + 1)).eval(strict_data_file_1) + assert should_read, "Should match: all values in range" + + +def test_strict_integer_lt_eq(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThanOrEqual("id", INT_MIN_VALUE - 1)).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: always false" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThanOrEqual("id", INT_MIN_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: 31 and greater not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThanOrEqual("id", INT_MAX_VALUE)).eval(strict_data_file_1) + assert should_read, "Should match: all values in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, LessThanOrEqual("id", INT_MAX_VALUE + 1)).eval( + strict_data_file_1 + ) + assert should_read, "Should match: all values in range" + + +def test_strict_integer_gt(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThan("id", INT_MAX_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: always false" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThan("id", INT_MAX_VALUE - 1)).eval(strict_data_file_1) + assert not should_read, "Should not match: 77 and less not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThan("id", INT_MIN_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: 30 not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThan("id", INT_MIN_VALUE - 1)).eval(strict_data_file_1) + assert should_read, "Should match: all values in range" + + +def test_strict_integer_gt_eq(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThanOrEqual("id", INT_MAX_VALUE + 1)).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: no values in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThanOrEqual("id", INT_MAX_VALUE)).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: 78 and lower are not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThanOrEqual("id", INT_MIN_VALUE + 1)).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: 30 not in range" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, GreaterThanOrEqual("id", INT_MIN_VALUE)).eval( + strict_data_file_1 + ) + assert should_read, "Should match: all values in range" + + +def test_strict_integer_eq(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("id", INT_MIN_VALUE - 25)).eval(strict_data_file_1) + assert not should_read, "Should not match: all values != 5" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("id", INT_MIN_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: some values != 30" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("id", INT_MAX_VALUE - 4)).eval(strict_data_file_1) + assert not should_read, "Should not match: some values != 75" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("id", INT_MAX_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: some values != 79" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("id", INT_MAX_VALUE + 1)).eval(strict_data_file_1) + assert not should_read, "Should not match: some values != 80" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, EqualTo("always_5", INT_MIN_VALUE - 25)).eval( + strict_data_file_1 + ) + assert should_read, "Should match: all values == 5" + + +def test_strict_integer_not_eq(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MIN_VALUE - 25)).eval(strict_data_file_1) + assert should_read, "Should match: no values == 5" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MIN_VALUE - 1)).eval(strict_data_file_1) + assert should_read, "Should match: no values == 39" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MIN_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 30" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MAX_VALUE - 4)).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 75" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MAX_VALUE)).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 79" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MAX_VALUE + 1)).eval(strict_data_file_1) + assert should_read, "Should match: no values == 80" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotEqualTo("id", INT_MAX_VALUE + 6)).eval(strict_data_file_1) + assert should_read, "Should read: no values == 85" + + +def test_strict_integer_not_eq_rewritten(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MIN_VALUE - 25))).eval( + strict_data_file_1 + ) + assert should_read, "Should match: no values == 5" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MIN_VALUE - 1))).eval(strict_data_file_1) + assert should_read, "Should match: no values == 39" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MIN_VALUE))).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 30" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MAX_VALUE - 4))).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 75" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MAX_VALUE))).eval(strict_data_file_1) + assert not should_read, "Should not match: some value may be == 79" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MAX_VALUE + 1))).eval(strict_data_file_1) + assert should_read, "Should match: no values == 80" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, Not(EqualTo("id", INT_MAX_VALUE + 6))).eval(strict_data_file_1) + assert should_read, "Should read: no values == 85" + + +def test_strict_integer_in(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: all values != 5 and != 6" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some values != 30 and != 31" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some values != 75 and != 76" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some values != 78 and != 79" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some values != 80 and != 81)" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("always_5", {5, 6})).eval(strict_data_file_1) + assert should_read, "Should match: all values == 5" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("all_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert not should_read, "Should not match: in on all nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("some_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert not should_read, "Should not match: in on some nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, In("no_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert not should_read, "Should not match: no_nulls field does not have bounds" + + +def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file_1: DataFile) -> None: + # should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MIN_VALUE - 25, INT_MIN_VALUE - 24})).eval(strict_data_file_1) + # assert should_read, "Should match: all values != 5 and != 6" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MIN_VALUE - 1, INT_MIN_VALUE})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some values may be == 30" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MAX_VALUE - 4, INT_MAX_VALUE - 3})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some value may be == 75 or == 76" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MAX_VALUE, INT_MAX_VALUE + 1})).eval( + strict_data_file_1 + ) + assert not should_read, "Should not match: some value may be == 79" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("id", {INT_MAX_VALUE + 1, INT_MAX_VALUE + 2})).eval( + strict_data_file_1 + ) + assert should_read, "Should match: no values == 80 or == 81" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("always_5", {5, 6})).eval(strict_data_file_1) + assert not should_read, "Should not match: all values == 5" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("all_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert should_read, "Should match: notIn on all nulls column" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("some_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert should_read, "Should match: notIn on some nulls column, 'bbb' > 'abc' and 'bbb' < 'def'" + + should_read = _StrictMetricsEvaluator(strict_data_file_schema, NotIn("no_nulls", {"abc", "def"})).eval(strict_data_file_1) + assert not should_read, "Should not match: no_nulls field does not have bounds" diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 309bd28c4c..c974fc8b52 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -758,7 +758,6 @@ def test_invalid_uuid_conversions() -> None: DecimalType(9, 2), StringType(), FixedType(1), - BinaryType(), ], ) @@ -882,6 +881,25 @@ def test_uuid_literal_initialization() -> None: assert test_uuid.bytes == uuid_literal.value +def test_uuid_to_fixed() -> None: + test_uuid = uuid.uuid4() + uuid_literal = literal(test_uuid) + fixed_literal = uuid_literal.to(FixedType(16)) + assert test_uuid.bytes == fixed_literal.value + with pytest.raises(TypeError) as e: + uuid_literal.to(FixedType(15)) + assert "Cannot convert UUIDLiteral into fixed[15], different length: 15 <> 16" in str(e.value) + assert isinstance(fixed_literal, FixedLiteral) # type: ignore + + +def test_uuid_to_binary() -> None: + test_uuid = uuid.uuid4() + uuid_literal = literal(test_uuid) + binary_literal = uuid_literal.to(BinaryType()) + assert test_uuid.bytes == binary_literal.value + assert isinstance(binary_literal, BinaryLiteral) # type: ignore + + # __ __ ___ # | \/ |_ _| _ \_ _ # | |\/| | || | _/ || | diff --git a/tests/integration/test_add_files.py b/tests/integration/test_add_files.py new file mode 100644 index 0000000000..7c17618280 --- /dev/null +++ b/tests/integration/test_add_files.py @@ -0,0 +1,417 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name + +from datetime import date +from typing import Optional + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from pyspark.sql import SparkSession + +from pyiceberg.catalog import Catalog +from pyiceberg.exceptions import NoSuchTableError +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.table import Table +from pyiceberg.transforms import BucketTransform, IdentityTransform, MonthTransform +from pyiceberg.types import ( + BooleanType, + DateType, + IntegerType, + NestedField, + StringType, +) + +TABLE_SCHEMA = Schema( + NestedField(field_id=1, name="foo", field_type=BooleanType(), required=False), + NestedField(field_id=2, name="bar", field_type=StringType(), required=False), + NestedField(field_id=4, name="baz", field_type=IntegerType(), required=False), + NestedField(field_id=10, name="qux", field_type=DateType(), required=False), +) + +ARROW_SCHEMA = pa.schema([ + ("foo", pa.bool_()), + ("bar", pa.string()), + ("baz", pa.int32()), + ("qux", pa.date32()), +]) + +ARROW_TABLE = pa.Table.from_pylist( + [ + { + "foo": True, + "bar": "bar_string", + "baz": 123, + "qux": date(2024, 3, 7), + } + ], + schema=ARROW_SCHEMA, +) + +ARROW_SCHEMA_WITH_IDS = pa.schema([ + pa.field('foo', pa.bool_(), nullable=False, metadata={"PARQUET:field_id": "1"}), + pa.field('bar', pa.string(), nullable=False, metadata={"PARQUET:field_id": "2"}), + pa.field('baz', pa.int32(), nullable=False, metadata={"PARQUET:field_id": "3"}), + pa.field('qux', pa.date32(), nullable=False, metadata={"PARQUET:field_id": "4"}), +]) + + +ARROW_TABLE_WITH_IDS = pa.Table.from_pylist( + [ + { + "foo": True, + "bar": "bar_string", + "baz": 123, + "qux": date(2024, 3, 7), + } + ], + schema=ARROW_SCHEMA_WITH_IDS, +) + +ARROW_SCHEMA_UPDATED = pa.schema([ + ("foo", pa.bool_()), + ("baz", pa.int32()), + ("qux", pa.date32()), + ("quux", pa.int32()), +]) + +ARROW_TABLE_UPDATED = pa.Table.from_pylist( + [ + { + "foo": True, + "baz": 123, + "qux": date(2024, 3, 7), + "quux": 234, + } + ], + schema=ARROW_SCHEMA_UPDATED, +) + + +def _create_table( + session_catalog: Catalog, identifier: str, format_version: int, partition_spec: Optional[PartitionSpec] = None +) -> Table: + try: + session_catalog.drop_table(identifier=identifier) + except NoSuchTableError: + pass + + tbl = session_catalog.create_table( + identifier=identifier, + schema=TABLE_SCHEMA, + properties={"format-version": str(format_version)}, + partition_spec=partition_spec if partition_spec else PartitionSpec(), + ) + + return tbl + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_unpartitioned_table(spark: SparkSession, session_catalog: Catalog, format_version: int) -> None: + identifier = f"default.unpartitioned_table_v{format_version}" + tbl = _create_table(session_catalog, identifier, format_version) + + file_paths = [f"s3://warehouse/default/unpartitioned/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table(ARROW_TABLE) + + # add the parquet files as data files + tbl.add_files(file_paths=file_paths) + + # NameMapping must have been set to enable reads + assert tbl.name_mapping() is not None + + rows = spark.sql( + f""" + SELECT added_data_files_count, existing_data_files_count, deleted_data_files_count + FROM {identifier}.all_manifests + """ + ).collect() + + assert [row.added_data_files_count for row in rows] == [5] + assert [row.existing_data_files_count for row in rows] == [0] + assert [row.deleted_data_files_count for row in rows] == [0] + + df = spark.table(identifier) + assert df.count() == 5, "Expected 5 rows" + for col in df.columns: + assert df.filter(df[col].isNotNull()).count() == 5, "Expected all 5 rows to be non-null" + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_unpartitioned_table_raises_file_not_found( + spark: SparkSession, session_catalog: Catalog, format_version: int +) -> None: + identifier = f"default.unpartitioned_raises_not_found_v{format_version}" + tbl = _create_table(session_catalog, identifier, format_version) + + file_paths = [f"s3://warehouse/default/unpartitioned_raises_not_found/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table(ARROW_TABLE) + + # add the parquet files as data files + with pytest.raises(FileNotFoundError): + tbl.add_files(file_paths=file_paths + ["s3://warehouse/default/unpartitioned_raises_not_found/unknown.parquet"]) + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_unpartitioned_table_raises_has_field_ids( + spark: SparkSession, session_catalog: Catalog, format_version: int +) -> None: + identifier = f"default.unpartitioned_raises_field_ids_v{format_version}" + tbl = _create_table(session_catalog, identifier, format_version) + + file_paths = [f"s3://warehouse/default/unpartitioned_raises_field_ids/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA_WITH_IDS) as writer: + writer.write_table(ARROW_TABLE_WITH_IDS) + + # add the parquet files as data files + with pytest.raises(NotImplementedError): + tbl.add_files(file_paths=file_paths) + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_unpartitioned_table_with_schema_updates( + spark: SparkSession, session_catalog: Catalog, format_version: int +) -> None: + identifier = f"default.unpartitioned_table_schema_updates_v{format_version}" + tbl = _create_table(session_catalog, identifier, format_version) + + file_paths = [f"s3://warehouse/default/unpartitioned_schema_updates/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table(ARROW_TABLE) + + # add the parquet files as data files + tbl.add_files(file_paths=file_paths) + + # NameMapping must have been set to enable reads + assert tbl.name_mapping() is not None + + with tbl.update_schema() as update: + update.add_column("quux", IntegerType()) + update.delete_column("bar") + + file_path = f"s3://warehouse/default/unpartitioned_schema_updates/v{format_version}/test-6.parquet" + # write parquet files + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA_UPDATED) as writer: + writer.write_table(ARROW_TABLE_UPDATED) + + # add the parquet files as data files + tbl.add_files(file_paths=[file_path]) + rows = spark.sql( + f""" + SELECT added_data_files_count, existing_data_files_count, deleted_data_files_count + FROM {identifier}.all_manifests + """ + ).collect() + + assert [row.added_data_files_count for row in rows] == [5, 1, 5] + assert [row.existing_data_files_count for row in rows] == [0, 0, 0] + assert [row.deleted_data_files_count for row in rows] == [0, 0, 0] + + df = spark.table(identifier) + assert df.count() == 6, "Expected 6 rows" + assert len(df.columns) == 4, "Expected 4 columns" + + for col in df.columns: + value_count = 1 if col == "quux" else 6 + assert df.filter(df[col].isNotNull()).count() == value_count, f"Expected {value_count} rows to be non-null" + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_partitioned_table(spark: SparkSession, session_catalog: Catalog, format_version: int) -> None: + identifier = f"default.partitioned_table_v{format_version}" + + partition_spec = PartitionSpec( + PartitionField(source_id=4, field_id=1000, transform=IdentityTransform(), name="baz"), + PartitionField(source_id=10, field_id=1001, transform=MonthTransform(), name="qux_month"), + spec_id=0, + ) + + tbl = _create_table(session_catalog, identifier, format_version, partition_spec) + + date_iter = iter([date(2024, 3, 7), date(2024, 3, 8), date(2024, 3, 16), date(2024, 3, 18), date(2024, 3, 19)]) + + file_paths = [f"s3://warehouse/default/partitioned/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table( + pa.Table.from_pylist( + [ + { + "foo": True, + "bar": "bar_string", + "baz": 123, + "qux": next(date_iter), + } + ], + schema=ARROW_SCHEMA, + ) + ) + + # add the parquet files as data files + tbl.add_files(file_paths=file_paths) + + # NameMapping must have been set to enable reads + assert tbl.name_mapping() is not None + + rows = spark.sql( + f""" + SELECT added_data_files_count, existing_data_files_count, deleted_data_files_count + FROM {identifier}.all_manifests + """ + ).collect() + + assert [row.added_data_files_count for row in rows] == [5] + assert [row.existing_data_files_count for row in rows] == [0] + assert [row.deleted_data_files_count for row in rows] == [0] + + df = spark.table(identifier) + assert df.count() == 5, "Expected 5 rows" + for col in df.columns: + assert df.filter(df[col].isNotNull()).count() == 5, "Expected all 5 rows to be non-null" + + partition_rows = spark.sql( + f""" + SELECT partition, record_count, file_count + FROM {identifier}.partitions + """ + ).collect() + + assert [row.record_count for row in partition_rows] == [5] + assert [row.file_count for row in partition_rows] == [5] + assert [(row.partition.baz, row.partition.qux_month) for row in partition_rows] == [(123, 650)] + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_bucket_partitioned_table_fails(spark: SparkSession, session_catalog: Catalog, format_version: int) -> None: + identifier = f"default.partitioned_table_bucket_fails_v{format_version}" + + partition_spec = PartitionSpec( + PartitionField(source_id=4, field_id=1000, transform=BucketTransform(num_buckets=3), name="baz_bucket_3"), + spec_id=0, + ) + + tbl = _create_table(session_catalog, identifier, format_version, partition_spec) + + int_iter = iter(range(5)) + + file_paths = [f"s3://warehouse/default/partitioned_table_bucket_fails/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table( + pa.Table.from_pylist( + [ + { + "foo": True, + "bar": "bar_string", + "baz": next(int_iter), + "qux": date(2024, 3, 7), + } + ], + schema=ARROW_SCHEMA, + ) + ) + + # add the parquet files as data files + with pytest.raises(ValueError) as exc_info: + tbl.add_files(file_paths=file_paths) + assert ( + "Cannot infer partition value from parquet metadata for a non-linear Partition Field: baz_bucket_3 with transform bucket[3]" + in str(exc_info.value) + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_add_files_to_partitioned_table_fails_with_lower_and_upper_mismatch( + spark: SparkSession, session_catalog: Catalog, format_version: int +) -> None: + identifier = f"default.partitioned_table_mismatch_fails_v{format_version}" + + partition_spec = PartitionSpec( + PartitionField(source_id=4, field_id=1000, transform=IdentityTransform(), name="baz"), + spec_id=0, + ) + + tbl = _create_table(session_catalog, identifier, format_version, partition_spec) + + file_paths = [f"s3://warehouse/default/partitioned_table_mismatch_fails/v{format_version}/test-{i}.parquet" for i in range(5)] + # write parquet files + for file_path in file_paths: + fo = tbl.io.new_output(file_path) + with fo.create(overwrite=True) as fos: + with pq.ParquetWriter(fos, schema=ARROW_SCHEMA) as writer: + writer.write_table( + pa.Table.from_pylist( + [ + { + "foo": True, + "bar": "bar_string", + "baz": 123, + "qux": date(2024, 3, 7), + }, + { + "foo": True, + "bar": "bar_string", + "baz": 124, + "qux": date(2024, 3, 7), + }, + ], + schema=ARROW_SCHEMA, + ) + ) + + # add the parquet files as data files + with pytest.raises(ValueError) as exc_info: + tbl.add_files(file_paths=file_paths) + assert ( + "Cannot infer partition value from parquet metadata as there are more than one partition values for Partition Field: baz. lower_value=123, upper_value=124" + in str(exc_info.value) + ) diff --git a/tests/integration/test_rest_schema.py b/tests/integration/test_rest_schema.py index 4c758e4c3e..7aeb1bccdc 100644 --- a/tests/integration/test_rest_schema.py +++ b/tests/integration/test_rest_schema.py @@ -672,9 +672,13 @@ def test_rename_simple(simple_table: Table) -> None: with simple_table.update_schema() as schema_update: schema_update.rename_column("foo", "vo") + with simple_table.transaction() as txn: + with txn.update_schema() as schema_update: + schema_update.rename_column("bar", "var") + assert simple_table.schema() == Schema( NestedField(field_id=1, name="vo", field_type=StringType(), required=False), - NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True), + NestedField(field_id=2, name="var", field_type=IntegerType(), required=True), NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), identifier_field_ids=[2], ) @@ -682,7 +686,7 @@ def test_rename_simple(simple_table: Table) -> None: # Check that the name mapping gets updated assert simple_table.name_mapping() == NameMapping([ MappedField(field_id=1, names=['foo', 'vo']), - MappedField(field_id=2, names=['bar']), + MappedField(field_id=2, names=['bar', 'var']), MappedField(field_id=3, names=['baz']), ]) diff --git a/tests/integration/test_writes.py b/tests/integration/test_writes.py index a3c603141b..5d6be0a7a4 100644 --- a/tests/integration/test_writes.py +++ b/tests/integration/test_writes.py @@ -15,12 +15,13 @@ # specific language governing permissions and limitations # under the License. # pylint:disable=redefined-outer-name +import math import os import time import uuid from datetime import date, datetime from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from urllib.parse import urlparse import pyarrow as pa @@ -135,15 +136,19 @@ def arrow_table_with_only_nulls(pa_schema: pa.Schema) -> pa.Table: return pa.Table.from_pylist([{}, {}], schema=pa_schema) -def _create_table(session_catalog: Catalog, identifier: str, properties: Properties, data: List[pa.Table]) -> Table: +def _create_table( + session_catalog: Catalog, identifier: str, properties: Properties, data: Optional[List[pa.Table]] = None +) -> Table: try: session_catalog.drop_table(identifier=identifier) except NoSuchTableError: pass tbl = session_catalog.create_table(identifier=identifier, schema=TABLE_SCHEMA, properties=properties) - for d in data: - tbl.append(d) + + if data: + for d in data: + tbl.append(d) return tbl @@ -566,12 +571,15 @@ def test_summaries_with_only_nulls( 'total-records': '2', } - assert summaries[0] == { - 'total-data-files': '0', - 'total-delete-files': '0', + assert summaries[2] == { + 'removed-files-size': '4239', 'total-equality-deletes': '0', - 'total-files-size': '0', 'total-position-deletes': '0', + 'deleted-data-files': '1', + 'total-delete-files': '0', + 'total-files-size': '0', + 'deleted-records': '2', + 'total-data-files': '0', 'total-records': '0', } @@ -705,3 +713,69 @@ def test_table_properties_raise_for_none_value( session_catalog, identifier, {"format-version": format_version, **property_with_none}, [arrow_table_with_null] ) assert "None type is not a supported value in properties: property_name" in str(exc_info.value) + + +@pytest.mark.integration +@pytest.mark.parametrize("format_version", [1, 2]) +def test_inspect_snapshots( + spark: SparkSession, session_catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int +) -> None: + identifier = "default.table_metadata_snapshots" + tbl = _create_table(session_catalog, identifier, properties={"format-version": format_version}) + + tbl.overwrite(arrow_table_with_null) + # should produce a DELETE entry + tbl.overwrite(arrow_table_with_null) + # Since we don't rewrite, this should produce a new manifest with an ADDED entry + tbl.append(arrow_table_with_null) + + df = tbl.inspect.snapshots() + + assert df.column_names == [ + 'committed_at', + 'snapshot_id', + 'parent_id', + 'operation', + 'manifest_list', + 'summary', + ] + + for committed_at in df['committed_at']: + assert isinstance(committed_at.as_py(), datetime) + + for snapshot_id in df['snapshot_id']: + assert isinstance(snapshot_id.as_py(), int) + + assert df['parent_id'][0].as_py() is None + assert df['parent_id'][1:] == df['snapshot_id'][:2] + + assert [operation.as_py() for operation in df['operation']] == ['append', 'overwrite', 'append'] + + for manifest_list in df['manifest_list']: + assert manifest_list.as_py().startswith("s3://") + + assert df['summary'][0].as_py() == [ + ('added-files-size', '5459'), + ('added-data-files', '1'), + ('added-records', '3'), + ('total-data-files', '1'), + ('total-delete-files', '0'), + ('total-records', '3'), + ('total-files-size', '5459'), + ('total-position-deletes', '0'), + ('total-equality-deletes', '0'), + ] + + lhs = spark.table(f"{identifier}.snapshots").toPandas() + rhs = df.to_pandas() + for column in df.column_names: + for left, right in zip(lhs[column].to_list(), rhs[column].to_list()): + if column == 'summary': + # Arrow returns a list of tuples, instead of a dict + right = dict(right) + + if isinstance(left, float) and math.isnan(left) and isinstance(right, float) and math.isnan(right): + # NaN != NaN in Python + continue + + assert left == right, f"Difference in column {column}: {left} != {right}" diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 33c254daed..b99febd6e2 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -28,8 +28,8 @@ import pytest from pyarrow.fs import FileType, LocalFileSystem -from pyiceberg.avro.resolver import ResolveError from pyiceberg.catalog.noop import NoopCatalog +from pyiceberg.exceptions import ResolveError from pyiceberg.expressions import ( AlwaysFalse, AlwaysTrue, diff --git a/tests/io/test_pyarrow_stats.py b/tests/io/test_pyarrow_stats.py index 01b844a43e..41f1432dbf 100644 --- a/tests/io/test_pyarrow_stats.py +++ b/tests/io/test_pyarrow_stats.py @@ -52,7 +52,7 @@ MetricsMode, PyArrowStatisticsCollector, compute_statistics_plan, - fill_parquet_file_metadata, + data_file_statistics_from_parquet_metadata, match_metrics_mode, parquet_path_to_id_mapping, schema_to_pyarrow, @@ -185,13 +185,12 @@ def test_record_count() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert datafile.record_count == 4 @@ -199,13 +198,12 @@ def test_value_counts() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 7 assert datafile.value_counts[1] == 4 @@ -221,13 +219,12 @@ def test_column_sizes() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.column_sizes) == 7 # these values are an artifact of how the write_table encodes the columns @@ -242,13 +239,12 @@ def test_null_and_nan_counts() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.null_value_counts) == 7 assert datafile.null_value_counts[1] == 1 @@ -270,13 +266,12 @@ def test_bounds() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.lower_bounds) == 2 assert datafile.lower_bounds[1].decode() == "aaaaaaaaaaaaaaaa" @@ -314,14 +309,13 @@ def test_metrics_mode_none() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "none" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 0 assert len(datafile.null_value_counts) == 0 @@ -334,14 +328,13 @@ def test_metrics_mode_counts() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "counts" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 7 assert len(datafile.null_value_counts) == 7 @@ -354,14 +347,13 @@ def test_metrics_mode_full() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "full" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 7 assert len(datafile.null_value_counts) == 7 @@ -380,14 +372,13 @@ def test_metrics_mode_non_default_trunc() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "truncate(2)" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 7 assert len(datafile.null_value_counts) == 7 @@ -406,15 +397,14 @@ def test_column_metrics_mode() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "truncate(2)" table_metadata.properties["write.metadata.metrics.column.strings"] = "none" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 6 assert len(datafile.null_value_counts) == 6 @@ -508,14 +498,13 @@ def test_metrics_primitive_types() -> None: metadata, table_metadata = construct_test_table_primitive_types() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "truncate(2)" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 12 assert len(datafile.null_value_counts) == 12 @@ -607,14 +596,13 @@ def test_metrics_invalid_upper_bound() -> None: metadata, table_metadata = construct_test_table_invalid_upper_bound() schema = get_current_schema(table_metadata) - datafile = DataFile() table_metadata.properties["write.metadata.metrics.default"] = "truncate(2)" - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert len(datafile.value_counts) == 4 assert len(datafile.null_value_counts) == 4 @@ -635,13 +623,12 @@ def test_offsets() -> None: metadata, table_metadata = construct_test_table() schema = get_current_schema(table_metadata) - datafile = DataFile() - fill_parquet_file_metadata( - datafile, - metadata, - compute_statistics_plan(schema, table_metadata.properties), - parquet_path_to_id_mapping(schema), + statistics = data_file_statistics_from_parquet_metadata( + parquet_metadata=metadata, + stats_columns=compute_statistics_plan(schema, table_metadata.properties), + parquet_column_mapping=parquet_path_to_id_mapping(schema), ) + datafile = DataFile(**statistics.to_serialized_dict()) assert datafile.split_offsets is not None assert len(datafile.split_offsets) == 1 diff --git a/tests/table/test_init.py b/tests/table/test_init.py index f734211510..f1191295f3 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -53,15 +53,17 @@ AssertLastAssignedPartitionId, AssertRefSnapshotId, AssertTableUUID, + CommitTableRequest, RemovePropertiesUpdate, SetDefaultSortOrderUpdate, SetPropertiesUpdate, SetSnapshotRefUpdate, StaticTable, Table, + TableIdentifier, UpdateSchema, _apply_table_update, - _check_schema, + _check_schema_compatible, _match_deletes_to_data_file, _TableMetadataUpdateContext, update_table_metadata, @@ -1031,7 +1033,7 @@ def test_schema_mismatch_type(table_schema_simple: Schema) -> None: """ with pytest.raises(ValueError, match=expected): - _check_schema(table_schema_simple, other_schema) + _check_schema_compatible(table_schema_simple, other_schema) def test_schema_mismatch_nullability(table_schema_simple: Schema) -> None: @@ -1052,7 +1054,7 @@ def test_schema_mismatch_nullability(table_schema_simple: Schema) -> None: """ with pytest.raises(ValueError, match=expected): - _check_schema(table_schema_simple, other_schema) + _check_schema_compatible(table_schema_simple, other_schema) def test_schema_mismatch_missing_field(table_schema_simple: Schema) -> None: @@ -1072,7 +1074,7 @@ def test_schema_mismatch_missing_field(table_schema_simple: Schema) -> None: """ with pytest.raises(ValueError, match=expected): - _check_schema(table_schema_simple, other_schema) + _check_schema_compatible(table_schema_simple, other_schema) def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: @@ -1086,7 +1088,21 @@ def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: expected = r"PyArrow table contains more columns: new_field. Update the schema first \(hint, use union_by_name\)." with pytest.raises(ValueError, match=expected): - _check_schema(table_schema_simple, other_schema) + _check_schema_compatible(table_schema_simple, other_schema) + + +def test_schema_downcast(table_schema_simple: Schema) -> None: + # large_string type is compatible with string type + other_schema = pa.schema(( + pa.field("foo", pa.large_string(), nullable=True), + pa.field("bar", pa.int32(), nullable=False), + pa.field("baz", pa.bool_(), nullable=True), + )) + + try: + _check_schema_compatible(table_schema_simple, other_schema) + except Exception: + pytest.fail("Unexpected Exception raised when calling `_check_schema`") def test_table_properties(example_table_metadata_v2: Dict[str, Any]) -> None: @@ -1113,3 +1129,13 @@ def test_table_properties_raise_for_none_value(example_table_metadata_v2: Dict[s with pytest.raises(ValidationError) as exc_info: TableMetadataV2(**example_table_metadata_v2) assert "None type is not a supported value in properties: property_name" in str(exc_info.value) + + +def test_serialize_commit_table_request() -> None: + request = CommitTableRequest( + requirements=(AssertTableUUID(uuid='4bfd18a3-74c6-478e-98b1-71c4c32f4163'),), + identifier=TableIdentifier(namespace=['a'], name='b'), + ) + + deserialized_request = CommitTableRequest.model_validate_json(request.model_dump_json()) + assert request == deserialized_request diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index 0cf17b11a2..b4e30a6b84 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -233,6 +233,11 @@ def test_new_table_metadata_with_explicit_v1_format() -> None: expected_spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="bar")) + expected_sort_order = SortOrder( + SortField(source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST), + order_id=1, + ) + expected = TableMetadataV1( location="s3://some_v1_location/", table_uuid=actual.table_uuid, @@ -250,20 +255,16 @@ def test_new_table_metadata_with_explicit_v1_format() -> None: snapshots=[], snapshot_log=[], metadata_log=[], - sort_orders=[ - SortOrder( - SortField( - source_id=1, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_LAST - ), - order_id=1, - ) - ], + sort_orders=[expected_sort_order], default_sort_order_id=1, refs={}, format_version=1, ) assert actual.model_dump() == expected.model_dump() + assert actual.schemas == [expected_schema] + assert actual.partition_specs == [expected_spec] + assert actual.sort_orders == [expected_sort_order] def test_invalid_format_version(example_table_metadata_v1: Dict[str, Any]) -> None: diff --git a/tests/table/test_partitioning.py b/tests/table/test_partitioning.py index cb60c9a8e5..d7425bc351 100644 --- a/tests/table/test_partitioning.py +++ b/tests/table/test_partitioning.py @@ -127,5 +127,5 @@ def test_partition_type(table_schema_simple: Schema) -> None: assert spec.partition_type(table_schema_simple) == StructType( NestedField(field_id=1000, name="str_truncate", field_type=StringType(), required=False), - NestedField(field_id=1001, name="int_bucket", field_type=IntegerType(), required=False), + NestedField(field_id=1001, name="int_bucket", field_type=IntegerType(), required=True), ) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 3591847ad6..e85ecce506 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -18,7 +18,17 @@ import pytest from pyiceberg.manifest import DataFile, DataFileContent, ManifestContent, ManifestFile +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema from pyiceberg.table.snapshots import Operation, Snapshot, SnapshotSummaryCollector, Summary, update_snapshot_summaries +from pyiceberg.transforms import IdentityTransform +from pyiceberg.typedef import Record +from pyiceberg.types import ( + BooleanType, + IntegerType, + NestedField, + StringType, +) @pytest.fixture @@ -137,26 +147,66 @@ def manifest_file() -> ManifestFile: ) -@pytest.fixture -def data_file() -> DataFile: - return DataFile( - content=DataFileContent.DATA, - record_count=100, - file_size_in_bytes=1234, - ) +@pytest.mark.integration +def test_snapshot_summary_collector(table_schema_simple: Schema) -> None: + ssc = SnapshotSummaryCollector() + + assert ssc.build() == {} + data_file = DataFile(content=DataFileContent.DATA, record_count=100, file_size_in_bytes=1234, partition=Record()) + ssc.add_file(data_file, schema=table_schema_simple) + + assert ssc.build() == { + 'added-data-files': '1', + 'added-files-size': '1234', + 'added-records': '100', + } + +@pytest.mark.integration +def test_snapshot_summary_collector_with_partition() -> None: + # Given -def test_snapshot_summary_collector(data_file: DataFile) -> None: ssc = SnapshotSummaryCollector() assert ssc.build() == {} + schema = Schema( + NestedField(field_id=1, name="bool_field", field_type=BooleanType(), required=False), + NestedField(field_id=2, name="string_field", field_type=StringType(), required=False), + NestedField(field_id=3, name="int_field", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=3, field_id=1001, transform=IdentityTransform(), name='int_field')) + data_file_1 = DataFile(content=DataFileContent.DATA, record_count=100, file_size_in_bytes=1234, partition=Record(int_field=1)) + data_file_2 = DataFile(content=DataFileContent.DATA, record_count=200, file_size_in_bytes=4321, partition=Record(int_field=2)) + # When + ssc.add_file(data_file=data_file_1, schema=schema, partition_spec=spec) + ssc.remove_file(data_file=data_file_1, schema=schema, partition_spec=spec) + ssc.remove_file(data_file=data_file_2, schema=schema, partition_spec=spec) + + # Then + assert ssc.build() == { + 'added-files-size': '1234', + 'removed-files-size': '5555', + 'added-data-files': '1', + 'deleted-data-files': '2', + 'added-records': '100', + 'deleted-records': '300', + 'changed-partition-count': '2', + } - ssc.add_file(data_file) + # When + ssc.set_partition_summary_limit(10) + # Then assert ssc.build() == { - 'added-data-files': '1', 'added-files-size': '1234', + 'removed-files-size': '5555', + 'added-data-files': '1', + 'deleted-data-files': '2', 'added-records': '100', + 'deleted-records': '300', + 'changed-partition-count': '2', + 'partitions.int_field=1': 'added-files-size=1234,removed-files-size=1234,added-data-files=1,deleted-data-files=1,added-records=100,deleted-records=100', + 'partitions.int_field=2': 'removed-files-size=4321,deleted-data-files=1,deleted-records=200', } diff --git a/tests/test_schema.py b/tests/test_schema.py index 7e10dd5b0d..90bc70652f 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -1600,3 +1600,19 @@ def test_union_with_pa_schema(primitive_fields: NestedField) -> None: ) assert new_schema == expected_schema + + +def test_arrow_schema() -> None: + base_schema = Schema( + NestedField(field_id=1, name="foo", field_type=StringType(), required=True), + NestedField(field_id=2, name="bar", field_type=IntegerType(), required=False), + NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False), + ) + + expected_schema = pa.schema([ + pa.field("foo", pa.string(), nullable=False), + pa.field("bar", pa.int32(), nullable=True), + pa.field("baz", pa.bool_(), nullable=True), + ]) + + assert base_schema.as_arrow() == expected_schema diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 4fea7739d1..4dc3d9819f 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -17,7 +17,7 @@ # pylint: disable=eval-used,protected-access,redefined-outer-name from datetime import date from decimal import Decimal -from typing import Any, Callable +from typing import Any, Callable, Optional from uuid import UUID import mmh3 as mmh3 @@ -30,32 +30,42 @@ ) from typing_extensions import Annotated -from pyiceberg import transforms from pyiceberg.expressions import ( BoundEqualTo, BoundGreaterThan, BoundGreaterThanOrEqual, BoundIn, + BoundIsNull, BoundLessThan, BoundLessThanOrEqual, + BoundLiteralPredicate, + BoundNotEqualTo, BoundNotIn, BoundNotNull, BoundNotStartsWith, BoundReference, BoundStartsWith, EqualTo, + GreaterThan, GreaterThanOrEqual, In, + LessThan, LessThanOrEqual, + LiteralPredicate, + NotEqualTo, NotIn, NotNull, NotStartsWith, Reference, + SetPredicate, StartsWith, + UnaryPredicate, + UnboundPredicate, ) from pyiceberg.expressions.literals import ( DateLiteral, DecimalLiteral, + LongLiteral, TimestampLiteral, literal, ) @@ -74,7 +84,7 @@ YearTransform, parse_transform, ) -from pyiceberg.typedef import UTF8 +from pyiceberg.typedef import UTF8, L from pyiceberg.types import ( BinaryType, BooleanType, @@ -438,7 +448,7 @@ def test_truncate_method(type_var: PrimitiveType, value: Any, expected_human_str def test_unknown_transform() -> None: - unknown_transform = transforms.UnknownTransform("unknown") # type: ignore + unknown_transform = UnknownTransform("unknown") # type: ignore assert str(unknown_transform) == str(eval(repr(unknown_transform))) with pytest.raises(AttributeError): unknown_transform.transform(StringType())("test") @@ -603,9 +613,7 @@ def bound_reference_decimal() -> BoundReference[Decimal]: @pytest.fixture def bound_reference_long() -> BoundReference[int]: - return BoundReference( - field=NestedField(1, "field", DecimalType(8, 2), required=False), accessor=Accessor(position=0, inner=None) - ) + return BoundReference(field=NestedField(1, "field", LongType(), required=False), accessor=Accessor(position=0, inner=None)) def test_projection_bucket_unary(bound_reference_str: BoundReference[str]) -> None: @@ -958,3 +966,845 @@ def test_projection_truncate_string_not_starts_with(bound_reference_str: BoundRe assert TruncateTransform(2).project( "name", BoundNotStartsWith(term=bound_reference_str, literal=literal("hello")) ) == NotStartsWith(term="name", literal=literal("he")) + + +def _test_projection(lhs: Optional[UnboundPredicate[L]], rhs: Optional[UnboundPredicate[L]]) -> None: + assert type(lhs) == type(lhs), f"Different classes: {type(lhs)} != {type(rhs)}" + if lhs is None and rhs is None: + # Both null + pass + elif isinstance(lhs, UnaryPredicate) and isinstance(rhs, UnaryPredicate): + # Nothing more to check + pass + elif isinstance(lhs, LiteralPredicate) and isinstance(rhs, LiteralPredicate): + assert lhs.literal == rhs.literal, f"Different literal: {lhs.literal} != {rhs.literal}" + elif isinstance(lhs, SetPredicate) and isinstance(rhs, SetPredicate): + assert lhs.literals == rhs.literals, f"Different literals: {lhs.literals} != {rhs.literals}" + else: + raise ValueError(f"Comparing unrelated: {lhs} <> {rhs}") + + +def test_month_projection_strict_epoch(bound_reference_date: BoundReference[int]) -> None: + date = literal("1970-01-01").to(DateType()) + transform: Transform[Any, int] = MonthTransform() + _test_projection( + transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(0)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(0)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(0)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-1)), # In Java this is human string 1970-01 + ) + _test_projection( + transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(0)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1969-12-31").to(DateType()) + _test_projection( + transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={DateLiteral(-1), DateLiteral(0)}), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_month_projection_strict_lower_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("2017-01-01").to(DateType()) # == 564 months since epoch + transform: Transform[Any, int] = MonthTransform() + _test_projection( + transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(564)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(564)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(564)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(563)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(564)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("2017-12-02").to(DateType()) + _test_projection( + transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(575), LongLiteral(564)}), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_negative_month_projection_strict_lower_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("1969-01-01").to(DateType()) # == 564 months since epoch + transform: Transform[Any, int] = MonthTransform() + _test_projection( + transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(-12)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(-12)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-12)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-13)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(-12)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1969-12-31").to(DateType()) + _test_projection( + transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(-1), LongLiteral(-12)}), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_month_projection_strict_upper_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("2017-12-31").to(DateType()) # == 575 months since epoch + transform: Transform[Any, int] = MonthTransform() + _test_projection( + transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(575)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(576)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(575)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(575)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(575)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("2017-01-01").to(DateType()) + _test_projection( + transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(575), LongLiteral(564)}), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_negative_month_projection_strict_upper_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("1969-12-31").to(DateType()) # == -1 month since epoch + transform: Transform[Any, int] = MonthTransform() + _test_projection( + transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(0)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-1)), + ) + _test_projection( + transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1969-11-01").to(DateType()) + _test_projection( + transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(-1), LongLiteral(-2)}), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_day_strict(bound_reference_date: BoundReference[int]) -> None: + date = literal("2017-01-01").to(DateType()) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(17167)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(17168)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(17167)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(17166)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(17167)), + ) + _test_projection( + lhs=DayTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("2017-12-31").to(DateType()) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(17531), LongLiteral(17167)}), + ) + _test_projection( + lhs=DayTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_day_negative_strict(bound_reference_date: BoundReference[int]) -> None: + date = literal("1969-12-30").to(DateType()) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(-2)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(-1)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(-2)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-3)), + ) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(-2)), + ) + _test_projection( + lhs=DayTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1969-12-28").to(DateType()) + _test_projection( + DayTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(-2), LongLiteral(-4)}), + ) + _test_projection( + lhs=DayTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_year_strict_lower_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("2017-01-01").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(46)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(47)), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("2016-12-31").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(46), LongLiteral(47)}), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_negative_year_strict_lower_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("1970-01-01").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(0)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(0)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(0)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-1)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(0)), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1969-12-31").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(-1), LongLiteral(0)}), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_year_strict_upper_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("2017-12-31").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(48)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(47)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(47)), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("2016-01-01").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(46), LongLiteral(47)}), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_negative_year_strict_upper_bound(bound_reference_date: BoundReference[int]) -> None: + date = literal("1969-12-31").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThan(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_date, literal=date)), + LessThan(term="name", literal=LongLiteral(0)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_date, literal=date)), + GreaterThan(term="name", literal=LongLiteral(-1)), + ) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_date, literal=date)), + NotEqualTo(term="name", literal=DateLiteral(-1)), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundEqualTo(term=bound_reference_date, literal=date)), rhs=None + ) + + another_date = literal("1970-01-01").to(DateType()) + _test_projection( + YearTransform().strict_project(name="name", pred=BoundNotIn(term=bound_reference_date, literals={date, another_date})), + NotIn(term="name", literals={LongLiteral(-1), LongLiteral(0)}), + ) + _test_projection( + lhs=YearTransform().strict_project(name="name", pred=BoundIn(term=bound_reference_date, literals={date, another_date})), + rhs=None, + ) + + +def test_strict_bucket_integer(bound_reference_long: BoundReference[int]) -> None: + value = literal(100) + transform: Transform[Any, int] = BucketTransform(num_buckets=10) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_long, literal=value)), + rhs=LessThan(term="name", literal=literal(6)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_long, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)), + rhs=None, + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundNotIn(term=bound_reference_long, literals={literal(100 - 1), value, literal(100 + 1)}) + ), + rhs=NotIn(term=Reference("name"), literals={6, 7, 8}), + ) + + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundIn(term=bound_reference_long, literals={literal(100 - 1), value, literal(100 + 1)}) + ), + rhs=None, + ) + + +def test_strict_bucket_decimal(bound_reference_decimal: BoundReference[int]) -> None: + value = literal(Decimal("100.00")) + transform: Transform[Any, int] = BucketTransform(num_buckets=10) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_decimal, literal=value)), + rhs=LessThan(term="name", literal=literal(2)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_decimal, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_decimal, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=None, + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_decimal, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=None, + ) + _test_projection( + lhs=transform.strict_project( + name="name", + pred=BoundNotIn( + term=bound_reference_decimal, literals={literal(Decimal("99.00")), value, literal(Decimal("101.00"))} + ), + ), + rhs=NotIn(term=Reference("name"), literals={2, 6}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", + pred=BoundIn(term=bound_reference_decimal, literals={literal(Decimal("99.00")), value, literal(Decimal("101.00"))}), + ), + rhs=None, + ) + + +def test_strict_bucket_string(bound_reference_str: BoundReference[int]) -> None: + value = literal("abcdefg") + transform: Transform[Any, int] = BucketTransform(num_buckets=10) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_str, literal=value)), + rhs=LessThan(term="name", literal=literal(4)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_str, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_str, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_str, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_str, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_str, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundNotIn(term=bound_reference_str, literals={literal("abcdefg"), literal("abcdefgabc")}) + ), + rhs=NotIn(term=Reference("name"), literals={4, 9}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundIn(term=bound_reference_str, literals={literal("abcdefg"), literal("abcdefgabc")}) + ), + rhs=None, + ) + + +def test_strict_bucket_bytes(bound_reference_binary: BoundReference[int]) -> None: + value = literal(str.encode("abcdefg")) + transform: Transform[Any, int] = BucketTransform(num_buckets=10) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_binary, literal=value)), + rhs=LessThan(term="name", literal=literal(4)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_binary, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_binary, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_binary, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_binary, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_binary, literal=value)), + rhs=None, + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundNotIn(term=bound_reference_binary, literals={value, literal(str.encode("abcdehij"))}) + ), + rhs=NotIn(term=Reference("name"), literals={4, 6}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundIn(term=bound_reference_binary, literals={value, literal(str.encode("abcdehij"))}) + ), + rhs=None, + ) + + +def test_strict_bucket_uuid(bound_reference_uuid: BoundReference[int]) -> None: + value = literal(UUID('12345678123456781234567812345678')) + transform: Transform[Any, int] = BucketTransform(num_buckets=10) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotEqualTo(term=bound_reference_uuid, literal=value)), + rhs=LessThan(term="name", literal=literal(1)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundEqualTo(term=bound_reference_uuid, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_uuid, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_uuid, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_uuid, literal=value)), rhs=None + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_uuid, literal=value)), + rhs=None, + ) + _test_projection( + lhs=transform.strict_project( + name="name", + pred=BoundNotIn(term=bound_reference_uuid, literals={value, literal(UUID('12345678123456781234567812345679'))}), + ), + rhs=NotIn(term=Reference("name"), literals={1, 4}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", + pred=BoundIn(term=bound_reference_uuid, literals={value, literal(UUID('12345678123456781234567812345679'))}), + ), + rhs=None, + ) + + +def test_strict_identity_projection(bound_reference_long: BoundReference[int]) -> None: + transform: Transform[Any, Any] = IdentityTransform() + predicates = [ + BoundNotNull(term=bound_reference_long), + BoundIsNull(term=bound_reference_long), + BoundLessThan(term=bound_reference_long, literal=literal(100)), + BoundLessThanOrEqual(term=bound_reference_long, literal=literal(101)), + BoundGreaterThan(term=bound_reference_long, literal=literal(102)), + BoundGreaterThanOrEqual(term=bound_reference_long, literal=literal(103)), + BoundEqualTo(term=bound_reference_long, literal=literal(104)), + BoundNotEqualTo(term=bound_reference_long, literal=literal(105)), + ] + for predicate in predicates: + if isinstance(predicate, BoundLiteralPredicate): + _test_projection( + lhs=transform.strict_project( + name="name", + pred=predicate, + ), + rhs=predicate.as_unbound(term=Reference("name"), literal=predicate.literal), + ) + else: + _test_projection( + lhs=transform.strict_project( + name="name", + pred=predicate, + ), + rhs=predicate.as_unbound(term=Reference("name")), + ) + + +def test_truncate_strict_integer_lower_bound(bound_reference_long: BoundReference[int]) -> None: + value = literal(100) + transform: Transform[Any, Any] = TruncateTransform(width=10) + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=value)), + rhs=LessThan(term=Reference("name"), literal=LongLiteral(100)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=value)), + rhs=LessThanOrEqual(term=Reference("name"), literal=LongLiteral(100)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=LongLiteral(100)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=LongLiteral(90)), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundNotIn(term=bound_reference_long, literals={literal(99), literal(100), literal(101)}) + ), + rhs=NotIn(term=Reference("name"), literals={literal(90), literal(100)}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundIn(term=bound_reference_long, literals={literal(99), literal(100), literal(101)}) + ), + rhs=None, + ) + + +def test_truncate_strict_integer_upper_bound(bound_reference_long: BoundReference[int]) -> None: + value = literal(99) + transform: Transform[Any, Any] = TruncateTransform(width=10) + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=value)), + rhs=LessThan(term=Reference("name"), literal=LongLiteral(90)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=value)), + rhs=LessThan(term=Reference("name"), literal=LongLiteral(100)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=LongLiteral(90)), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=LongLiteral(90)), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundNotIn(term=bound_reference_long, literals={literal(99), literal(100), literal(101)}) + ), + rhs=NotIn(term=Reference("name"), literals={literal(90), literal(100)}), + ) + _test_projection( + lhs=transform.strict_project( + name="name", pred=BoundIn(term=bound_reference_long, literals={literal(99), literal(100), literal(101)}) + ), + rhs=None, + ) + + +def test_truncate_strict_decimal_lower_bound(bound_reference_decimal: BoundReference[Decimal]) -> None: + value = literal(Decimal("100.00")) + transform: Transform[Any, Any] = TruncateTransform(width=10) + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_decimal, literal=value)), + rhs=LessThan(term=Reference("name"), literal=Decimal("100.00")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=LessThanOrEqual(term=Reference("name"), literal=Decimal("100.00")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_decimal, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=Decimal("100.00")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=Decimal("99.90")), + ) + set_of_literals = {literal(Decimal("99.00")), literal(Decimal("100.00")), literal(Decimal("101.00"))} + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_decimal, literals=set_of_literals)), + rhs=NotIn(term=Reference("name"), literals=set_of_literals), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_decimal, literals=set_of_literals)), rhs=None + ) + + +def test_truncate_strict_decimal_upper_bound(bound_reference_decimal: BoundReference[Decimal]) -> None: + value = literal(Decimal("99.99")) + transform: Transform[Any, Any] = TruncateTransform(width=10) + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_decimal, literal=value)), + rhs=LessThan(term=Reference("name"), literal=Decimal("99.90")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=LessThan(term=Reference("name"), literal=Decimal("100.00")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_decimal, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=Decimal("99.90")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_decimal, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=Decimal("99.90")), + ) + set_of_literals = {literal(Decimal("98.99")), literal(Decimal("99.99")), literal(Decimal("100.99"))} + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_decimal, literals=set_of_literals)), + rhs=NotIn( + term=Reference("name"), literals={literal(Decimal("98.90")), literal(Decimal("99.90")), literal(Decimal("100.90"))} + ), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_decimal, literals=set_of_literals)), rhs=None + ) + + +def test_string_strict(bound_reference_str: BoundReference[str]) -> None: + value = literal("abcdefg") + transform: Transform[Any, Any] = TruncateTransform(width=5) + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_str, literal=value)), + rhs=LessThan(term=Reference("name"), literal=literal("abcde")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_str, literal=value)), + rhs=LessThan(term=Reference("name"), literal=literal("abcde")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_str, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=literal("abcde")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_str, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=literal("abcde")), + ) + set_of_literals = {literal("abcde"), literal("abcdefg")} + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_str, literals=set_of_literals)), + rhs=NotEqualTo(term=Reference("name"), literal=literal("abcde")), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_str, literals=set_of_literals)), rhs=None + ) + + +def test_strict_binary(bound_reference_binary: BoundReference[str]) -> None: + value = literal(b"abcdefg") + transform: Transform[Any, Any] = TruncateTransform(width=5) + abcde = literal(b"abcde") + + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_binary, literal=value)), + rhs=LessThan(term=Reference("name"), literal=abcde), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_binary, literal=value)), + rhs=LessThan(term=Reference("name"), literal=abcde), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_binary, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=abcde), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_binary, literal=value)), + rhs=GreaterThan(term=Reference("name"), literal=abcde), + ) + set_of_literals = {literal(b"abcde"), literal(b"abcdefg")} + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundNotIn(term=bound_reference_binary, literals=set_of_literals)), + rhs=NotEqualTo(term=Reference("name"), literal=abcde), + ) + _test_projection( + lhs=transform.strict_project(name="name", pred=BoundIn(term=bound_reference_binary, literals=set_of_literals)), rhs=None + )