From 0d0175909f0be83176af7d1c8aaebf937e5ca6df Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Mon, 16 Dec 2024 17:34:32 +0300 Subject: [PATCH 1/4] - Added the namespace_exists function in the RESTCatalog - Added the relevant unit tests --- pyiceberg/catalog/rest.py | 27 +++++++++++++++++++++++ tests/catalog/test_rest.py | 45 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index 287c5754a9..f56c1daf10 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -94,6 +94,7 @@ class Endpoints: load_namespace_metadata: str = "namespaces/{namespace}" drop_namespace: str = "namespaces/{namespace}" update_namespace_properties: str = "namespaces/{namespace}/properties" + namespace_exists: str = "namespaces/{namespace}" list_tables: str = "namespaces/{namespace}/tables" create_table: str = "namespaces/{namespace}/tables" register_table = "namespaces/{namespace}/register" @@ -869,6 +870,32 @@ def update_namespace_properties( updated=parsed_response.updated, missing=parsed_response.missing, ) + + @retry(**_RETRY_ARGS) + def namespace_exists(self, namespace: Union[str, Identifier]) -> bool: + """Check if a namespace exists. + + Args: + identifier (str | Identifier): namespace identifier. + + Returns: + bool: True if the namespace exists, False otherwise. + """ + namespace_tuple = self._check_valid_namespace_identifier(namespace) + namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) + response = self._session.head(self.url(Endpoints.namespace_exists, namespace=namespace)) + + if response.status_code == 404: + return False + elif response.status_code in (200, 204): + return True + + try: + response.raise_for_status() + except HTTPError as exc: + self._handle_non_200_response(exc, {}) + + return False @retry(**_RETRY_ARGS) def table_exists(self, identifier: Union[str, Identifier]) -> bool: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 5c6d402842..091a67166b 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -681,6 +681,51 @@ def test_update_namespace_properties_200(rest_mock: Mocker) -> None: assert response == PropertiesUpdateSummary(removed=[], updated=["prop"], missing=["abc"]) +def test_namespace_exists_200(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko", + status_code=200, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + assert catalog.namespace_exists("fokko") + + +def test_namespace_exists_204(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko", + status_code=204, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + assert catalog.namespace_exists("fokko") + + +def test_namespace_exists_404(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko", + status_code=404, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + assert not catalog.namespace_exists("fokko") + + +def test_namespace_exists_500(rest_mock: Mocker) -> None: + rest_mock.head( + f"{TEST_URI}v1/namespaces/fokko", + status_code=500, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + with pytest.raises(ServerError): + catalog.namespace_exists("fokko") + + def test_update_namespace_properties_404(rest_mock: Mocker) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/properties", From d1e643c35157e13b1b4b81c6e40a60a17a39e434 Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Mon, 16 Dec 2024 18:53:53 +0300 Subject: [PATCH 2/4] - Removed docstring to match other namespace functions --- pyiceberg/catalog/rest.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index f56c1daf10..e3ea5e7874 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -870,17 +870,9 @@ def update_namespace_properties( updated=parsed_response.updated, missing=parsed_response.missing, ) - + @retry(**_RETRY_ARGS) def namespace_exists(self, namespace: Union[str, Identifier]) -> bool: - """Check if a namespace exists. - - Args: - identifier (str | Identifier): namespace identifier. - - Returns: - bool: True if the namespace exists, False otherwise. - """ namespace_tuple = self._check_valid_namespace_identifier(namespace) namespace = NAMESPACE_SEPARATOR.join(namespace_tuple) response = self._session.head(self.url(Endpoints.namespace_exists, namespace=namespace)) From ceffe08ad90a0d150c6f1a24f8c0a75d9f7c7850 Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 17 Dec 2024 21:39:16 +0300 Subject: [PATCH 3/4] - Added integration test for REST Catalog namespace_exists functionality --- tests/integration/test_rest_catalog.py | 45 ++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/integration/test_rest_catalog.py diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py new file mode 100644 index 0000000000..67424d9ef1 --- /dev/null +++ b/tests/integration/test_rest_catalog.py @@ -0,0 +1,45 @@ +import pytest + +from pyiceberg.catalog.rest import RestCatalog + +TEST_NAMESPACE_IDENTIFIER = "TEST NS" + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) +def test_namespace_exists(catalog: RestCatalog) -> None: + if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) +def test_namespace_not_exists(catalog: RestCatalog) -> None: + if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): + catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + + assert not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) +def test_create_namespace_if_not_exists(catalog: RestCatalog) -> None: + if catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): + catalog.drop_namespace(TEST_NAMESPACE_IDENTIFIER) + + catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER) + + assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) + + +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog")]) +def test_create_namespace_if_already_existing(catalog: RestCatalog) -> None: + if not catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER): + catalog.create_namespace(TEST_NAMESPACE_IDENTIFIER) + + catalog.create_namespace_if_not_exists(TEST_NAMESPACE_IDENTIFIER) + + assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) From fcd11116d9c830b91be6c840cb2454acf7856ac1 Mon Sep 17 00:00:00 2001 From: Ahmed Nader Date: Tue, 17 Dec 2024 23:18:45 +0300 Subject: [PATCH 4/4] - Added ASF license to test_rest_catalog.py to recover from failing test --- tests/integration/test_rest_catalog.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index 67424d9ef1..24a8d9f6ef 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -1,3 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint:disable=redefined-outer-name + import pytest from pyiceberg.catalog.rest import RestCatalog