Skip to content

Commit cc4e8a3

Browse files
formanforman
authoredMar 5, 2024··
Merge pull request #936 from dcs4cop/forman-919-ds_iterator_type
Data stores can now return data iterators
·
v1.8.3v1.4.1
2 parents c149bcf + 404b873 commit cc4e8a3

File tree

3 files changed

+183
-110
lines changed

3 files changed

+183
-110
lines changed
 

‎CHANGES.md

+7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22

33
### Enhancements
44

5+
* Data stores can now return _data iterators_ from their `open_data()` method.
6+
For example, a data store implementation can now return a data cube either
7+
with a time dimension of size 100, or could be asked to return 100 cube
8+
time slices with dimension size 1 in form of an iterator.
9+
This feature has been added to effectively support the new
10+
[zappend](https://github.com/bcdev/zappend) tool. (#919)
11+
512
### Fixes
613

714
### Other changes

‎test/core/store/test_datatype.py

+110-59
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,16 @@
55
import xarray
66

77
from xcube.core.mldataset import MultiLevelDataset
8-
from xcube.core.store.datatype import ANY_TYPE, DataTypeLike
8+
from xcube.core.store.datatype import ANY_TYPE
99
from xcube.core.store.datatype import DATASET_TYPE
10+
from xcube.core.store.datatype import DATASET_ITERATOR_TYPE
11+
from xcube.core.store.datatype import DataIterator
1012
from xcube.core.store.datatype import DataType
13+
from xcube.core.store.datatype import DataTypeLike
1114
from xcube.core.store.datatype import GEO_DATA_FRAME_TYPE
15+
from xcube.core.store.datatype import GEO_DATA_FRAME_ITERATOR_TYPE
1216
from xcube.core.store.datatype import MULTI_LEVEL_DATASET_TYPE
17+
from xcube.core.store.datatype import MULTI_LEVEL_DATASET_ITERATOR_TYPE
1318
from xcube.util.jsonschema import JsonStringSchema
1419

1520

@@ -27,87 +32,133 @@ class C:
2732

2833
class DataTypeTest(unittest.TestCase):
2934
def test_normalize_to_any(self):
30-
self.assertNormalizeOk('any',
31-
ANY_TYPE,
32-
object,
33-
'any',
34-
('any', '*', 'object', 'builtins.object'))
35-
self.assertNormalizeOk(None,
36-
ANY_TYPE,
37-
object,
38-
'any',
39-
('any', '*', 'object', 'builtins.object'))
40-
self.assertNormalizeOk(type(None),
41-
ANY_TYPE,
42-
object,
43-
'any',
44-
('any', '*', 'object', 'builtins.object'))
35+
self.assertNormalizeOk(
36+
"any", ANY_TYPE, object, "any", ("any", "*", "object", "builtins.object")
37+
)
38+
self.assertNormalizeOk(
39+
None, ANY_TYPE, object, "any", ("any", "*", "object", "builtins.object")
40+
)
41+
self.assertNormalizeOk(
42+
type(None),
43+
ANY_TYPE,
44+
object,
45+
"any",
46+
("any", "*", "object", "builtins.object"),
47+
)
4548

4649
def test_normalize_to_dataset(self):
47-
self.assertNormalizeOk('dataset',
48-
DATASET_TYPE,
49-
xarray.Dataset,
50-
'dataset',
51-
('dataset',
52-
'xarray.Dataset',
53-
'xarray.core.dataset.Dataset'))
50+
self.assertNormalizeOk(
51+
"dataset",
52+
DATASET_TYPE,
53+
xarray.Dataset,
54+
"dataset",
55+
("dataset", "xarray.Dataset", "xarray.core.dataset.Dataset"),
56+
)
5457

5558
def test_normalize_to_mldataset(self):
56-
self.assertNormalizeOk('mldataset',
57-
MULTI_LEVEL_DATASET_TYPE,
58-
MultiLevelDataset,
59-
'mldataset',
60-
('mldataset',
61-
'xcube.MultiLevelDataset',
62-
'xcube.core.mldataset.MultiLevelDataset',
63-
'xcube.core.mldataset.abc.MultiLevelDataset'))
59+
self.assertNormalizeOk(
60+
"mldataset",
61+
MULTI_LEVEL_DATASET_TYPE,
62+
MultiLevelDataset,
63+
"mldataset",
64+
(
65+
"mldataset",
66+
"xcube.MultiLevelDataset",
67+
"xcube.core.mldataset.MultiLevelDataset",
68+
"xcube.core.mldataset.abc.MultiLevelDataset",
69+
),
70+
)
6471

6572
def test_normalize_to_geodataframe(self):
66-
self.assertNormalizeOk('geodataframe',
67-
GEO_DATA_FRAME_TYPE,
68-
geopandas.GeoDataFrame,
69-
'geodataframe',
70-
('geodataframe',
71-
'geopandas.GeoDataFrame',
72-
'geopandas.geodataframe.GeoDataFrame'))
73-
74-
def assertNormalizeOk(self,
75-
data_type: DataTypeLike,
76-
expected_data_type,
77-
expected_dtype,
78-
expected_alias,
79-
expected_aliases):
73+
self.assertNormalizeOk(
74+
"geodataframe",
75+
GEO_DATA_FRAME_TYPE,
76+
geopandas.GeoDataFrame,
77+
"geodataframe",
78+
(
79+
"geodataframe",
80+
"geopandas.GeoDataFrame",
81+
"geopandas.geodataframe.GeoDataFrame",
82+
),
83+
)
84+
85+
def test_normalize_to_dataset_iterator(self):
86+
self.assertNormalizeOk(
87+
"dsiter",
88+
DATASET_ITERATOR_TYPE,
89+
DataIterator,
90+
"dsiter",
91+
(
92+
"dsiter",
93+
"DataIterator[xarray.Dataset]",
94+
"xcube.core.store.datatype.DataIterator",
95+
),
96+
)
97+
98+
def test_normalize_to_ml_dataset_iterator(self):
99+
self.assertNormalizeOk(
100+
"mldsiter",
101+
MULTI_LEVEL_DATASET_ITERATOR_TYPE,
102+
DataIterator,
103+
"mldsiter",
104+
(
105+
"mldsiter",
106+
"DataIterator[xcube.MultiLevelDataset]",
107+
"xcube.core.store.datatype.DataIterator",
108+
),
109+
)
110+
111+
def test_normalize_to_gdf_iterator(self):
112+
self.assertNormalizeOk(
113+
"gdfiter",
114+
GEO_DATA_FRAME_ITERATOR_TYPE,
115+
DataIterator,
116+
"gdfiter",
117+
(
118+
"gdfiter",
119+
"DataIterator[geopandas.GeoDataFrame]",
120+
"xcube.core.store.datatype.DataIterator",
121+
),
122+
)
123+
124+
def assertNormalizeOk(
125+
self,
126+
data_type: DataTypeLike,
127+
expected_data_type,
128+
expected_dtype,
129+
expected_alias,
130+
expected_aliases,
131+
):
80132
data_type = DataType.normalize(data_type)
81133
self.assertIs(expected_data_type, data_type)
82134
self.assertIs(expected_dtype, data_type.dtype)
83135
self.assertEqual(expected_alias, data_type.alias)
84136
self.assertEqual(expected_alias, str(data_type))
85-
self.assertEqual(f'{expected_alias!r}', repr(data_type))
137+
self.assertEqual(f"{expected_alias!r}", repr(data_type))
86138
self.assertEqual(expected_aliases, data_type.aliases)
87-
for other_alias in data_type.aliases:
88-
self.assertIs(data_type,
89-
DataType.normalize(other_alias))
90-
self.assertIs(expected_data_type,
91-
DataType.normalize(expected_dtype))
92-
self.assertIs(expected_data_type,
93-
DataType.normalize(expected_data_type))
139+
self.assertIs(expected_data_type, DataType.normalize(expected_data_type))
140+
# The following tests are not applicable to iterators,
141+
# because iterators are not (cannot be) forced to have
142+
# a specific item type.
143+
if data_type.dtype is not DataIterator:
144+
for other_alias in data_type.aliases:
145+
self.assertIs(data_type, DataType.normalize(other_alias))
146+
self.assertIs(expected_data_type, DataType.normalize(expected_dtype))
94147

95148
def test_normalize_non_default_types(self):
96149
data_type = DataType.normalize(str)
97150
self.assertIs(str, data_type.dtype)
98-
self.assertEqual('builtins.str', data_type.alias)
151+
self.assertEqual("builtins.str", data_type.alias)
99152

100153
def test_normalize_failure(self):
101154
with self.assertRaises(ValueError) as cm:
102-
DataType.normalize('Gnartz')
103-
self.assertEqual("unknown data type 'Gnartz'",
104-
f'{cm.exception}')
155+
DataType.normalize("Gnartz")
156+
self.assertEqual("unknown data type 'Gnartz'", f"{cm.exception}")
105157

106158
with self.assertRaises(ValueError) as cm:
107159
# noinspection PyTypeChecker
108160
DataType.normalize(42)
109-
self.assertEqual("cannot convert 42 into a data type",
110-
f'{cm.exception}')
161+
self.assertEqual("cannot convert 42 into a data type", f"{cm.exception}")
111162

112163
def test_equality(self):
113164
self.assertIs(DATASET_TYPE, DATASET_TYPE)

0 commit comments

Comments
 (0)
Please sign in to comment.