merge master

Author: Diego Ardila

nucleus/dataset.py

@@ -22,7 +22,11 @@
     REQUEST_ID_KEY,
     UPDATE_KEY,
 )
-from .dataset_item import DatasetItem, check_all_paths_remote
+from .dataset_item import (
+    DatasetItem,
+    check_all_paths_remote,
+    check_for_duplicate_reference_ids,
+)
 from .payload_constructor import construct_model_run_creation_payload
 
 
@@ -191,6 +195,8 @@ def append(
             'ignored_items': int,
         }
         """
+        check_for_duplicate_reference_ids(dataset_items)
+
         if asynchronous:
             check_all_paths_remote(dataset_items)
             request_id = uuid.uuid4().hex
@@ -199,8 +205,6 @@ def append(
                 route=f"dataset/{self.id}/signedUrl/{request_id}",
                 requests_command=requests.get,
             )
-            print(response["signed_url"])
-
             serialize_and_write_to_presigned_url(
                 dataset_items, response["signed_url"]
             )

nucleus/dataset_item.py

@@ -1,7 +1,8 @@
+from collections import Counter
 import json
 import os.path
 from dataclasses import dataclass
-from typing import List, Optional
+from typing import Optional, Sequence
 
 from .constants import (
     DATASET_ITEM_ID_KEY,
@@ -58,10 +59,26 @@ def is_local_path(path: str) -> bool:
     return path_components[0] not in {"https:", "http:", "s3:", "gs:"}
 
 
-def check_all_paths_remote(dataset_items: List[DatasetItem]):
+def check_all_paths_remote(dataset_items: Sequence[DatasetItem]):
     for item in dataset_items:
         if is_local_path(item.image_location):
             raise ValueError(
                 f"All paths must be remote, but {item.image_location} is either "
                 "local, or a remote URL type that is not supported."
             )
+
+
+def check_for_duplicate_reference_ids(dataset_items: Sequence[DatasetItem]):
+    ref_ids = []
+    for dataset_item in dataset_items:
+        if dataset_item.reference_id is not None:
+            ref_ids.append(dataset_item.reference_id)
+    if len(ref_ids) != len(set(ref_ids)):
+        duplicates = {
+            f"{key}": f"Count: {value}"
+            for key, value in Counter(ref_ids).items()
+        }
+        raise ValueError(
+            "Duplicate reference ids found among dataset_items: %s"
+            % duplicates
+        )

tests/test_annotation.py

@@ -57,7 +57,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_box_gt_upload(dataset):

tests/test_dataset.py

@@ -96,7 +96,7 @@ def test_dataset_create_and_delete(CLIENT):
 
     # Deletion
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_dataset_append(dataset):
@@ -204,6 +204,22 @@ def test_dataset_list_autotags(CLIENT, dataset):
     assert autotag_response == []
 
 
+def test_raises_error_for_duplicate():
+    fake_dataset = Dataset("fake", NucleusClient("fake"))
+    with pytest.raises(ValueError) as error:
+        fake_dataset.append(
+            [
+                DatasetItem("fake", "duplicate"),
+                DatasetItem("fake", "duplicate"),
+            ]
+        )
+    assert (
+        str(error.value)
+        == "Duplicate reference ids found among dataset_items:"
+        " {'duplicate': 'Count: 2'}"
+    )
+
+
 def test_dataset_export_autotag_scores(CLIENT):
     # This test can only run for the test user who has an indexed dataset.
     # TODO: if/when we can create autotags via api, create one instead.

tests/test_indexing.py

@@ -34,7 +34,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_index_integration(dataset):

tests/test_prediction.py

@@ -65,7 +65,7 @@ def model_run(CLIENT):
     yield run
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
     response = CLIENT.delete_model(model.id)
     assert response == {}
 

tests/test_slice.py

@@ -16,7 +16,7 @@ def dataset(CLIENT):
     yield ds
 
     response = CLIENT.delete_dataset(ds.id)
-    assert response == {}
+    assert response == {"message": "Beginning dataset deletion..."}
 
 
 def test_reprs():