Skip to content

Commit c58a262

Browse files
anesson-csanesson-cs
committed
test: add tests for download_all() method with 'exhaust' parameter set to 'True'
1 parent 78cbc67 commit c58a262

File tree

1 file changed

+198
-0
lines changed

1 file changed

+198
-0
lines changed

‎tests/units/test_download_plugins.py‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import tarfile
2323
import unittest
2424
import zipfile
25+
from copy import copy
2526
from itertools import chain
2627
from pathlib import Path
2728
from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir
@@ -42,10 +43,13 @@
4243
OFFLINE_STATUS,
4344
ONLINE_STATUS,
4445
USER_AGENT,
46+
EODataAccessGateway,
4547
EOProduct,
48+
FilterProperty,
4649
HTTPDownload,
4750
NotAvailableError,
4851
PluginManager,
52+
SearchResult,
4953
config,
5054
load_default_config,
5155
override_config_from_mapping,
@@ -162,6 +166,200 @@ def test_plugins_download_base_prepare_download_dir_permission(self):
162166
)
163167
self.assertIn("Unable to create records directory", str(cm.output))
164168

169+
@mock.patch("eodag.api.product._product.EOProduct.download", autospec=True)
170+
@mock.patch("eodag.api.search_result.SearchResult.crunch", autospec=True)
171+
@mock.patch("eodag.api.core.EODataAccessGateway._do_search", autospec=True)
172+
def test_plugins_download_base_download_all_exhaust_ok(
173+
self, mock__do_search, mock_crunch, mock_download
174+
):
175+
"""Download.download_all with "exhaust" parameter set to True must download
176+
products from all pages of the initial product search request"""
177+
search_iter_page_search_result_1 = SearchResult(
178+
[
179+
EOProduct(
180+
"peps",
181+
dict(
182+
geometry="POINT (0 0)",
183+
title="other_dummy_product_1",
184+
id="other_dummy_1",
185+
cloudCover=80,
186+
),
187+
)
188+
]
189+
)
190+
search_iter_page_search_result_1.search_kwargs = {
191+
"page": 2,
192+
"items_per_page": 1,
193+
"productType": "S2_MSI_L1C",
194+
"geometry": None,
195+
"cloudCover": 80,
196+
}
197+
search_iter_page_search_result_2 = SearchResult(
198+
[
199+
EOProduct(
200+
"peps",
201+
dict(
202+
geometry="POINT (0 0)",
203+
title="other_dummy_product_2",
204+
id="other_dummy_2",
205+
cloudCover=20,
206+
),
207+
)
208+
]
209+
)
210+
search_iter_page_search_result_2.search_kwargs = {
211+
"page": 3,
212+
"items_per_page": 1,
213+
"productType": "S2_MSI_L1C",
214+
"geometry": None,
215+
"cloudCover": 80,
216+
}
217+
search_iter_page_search_result_3 = SearchResult([])
218+
search_iter_page_search_result_3.search_kwargs = {
219+
"page": 4,
220+
"items_per_page": 1,
221+
"productType": "S2_MSI_L1C",
222+
"geometry": None,
223+
"cloudCover": 80,
224+
}
225+
226+
do_search_results_list = [
227+
search_iter_page_search_result_1,
228+
search_iter_page_search_result_2,
229+
search_iter_page_search_result_3,
230+
]
231+
non_empty_do_search_results = SearchResult(
232+
[
233+
do_search_result[0]
234+
for do_search_result in do_search_results_list
235+
if do_search_result
236+
]
237+
)
238+
239+
mock__do_search.side_effect = do_search_results_list
240+
mock_crunch.return_value = search_iter_page_search_result_2
241+
242+
search_result = SearchResult([self.product])
243+
# simulate search kwargs and crunchers of the search result
244+
search_result.search_kwargs = {
245+
"page": 1,
246+
"items_per_page": 1,
247+
"productType": "S2_MSI_L1C",
248+
"geometry": None,
249+
"cloudCover": 80,
250+
}
251+
search_result.crunchers = [FilterProperty({"cloudCover": 20, "operator": "lt"})]
252+
# save a copy of the search result to check if it was called well below
253+
tmp_search_result = copy(search_result)
254+
255+
dag = EODataAccessGateway()
256+
257+
plugin = self.get_download_plugin(self.product)
258+
259+
with self.assertLogs(level="DEBUG") as cm:
260+
paths = plugin.download_all(search_result, dag, exhaust=True)
261+
262+
# check that _do_search() method is called for all pages except the one already requested
263+
self.assertEqual(mock__do_search.call_count, 3)
264+
default_kwargs = {"count": False, "raise_errors": True}
265+
self.assertDictEqual(
266+
{
267+
**search_result.search_kwargs,
268+
**default_kwargs,
269+
**{"page": search_result.search_kwargs["page"] + 1},
270+
},
271+
mock__do_search.call_args_list[0][1],
272+
)
273+
self.assertDictEqual(
274+
{
275+
**search_result.search_kwargs,
276+
**default_kwargs,
277+
**{"page": search_result.search_kwargs["page"] + 2},
278+
},
279+
mock__do_search.call_args_list[1][1],
280+
)
281+
self.assertDictEqual(
282+
{
283+
**search_result.search_kwargs,
284+
**default_kwargs,
285+
**{"page": search_result.search_kwargs["page"] + 3},
286+
},
287+
mock__do_search.call_args_list[2][1],
288+
)
289+
# check that the initial dag is called in each search request
290+
self.assertSetEqual(
291+
{dag}, {call_args[0][0] for call_args in mock__do_search.call_args_list}
292+
)
293+
294+
# chat that crunch() method is called with search_iter_page() results and for the cruncher given
295+
self.assertEqual(mock_crunch.call_count, 1)
296+
self.assertTupleEqual(
297+
(non_empty_do_search_results, search_result.crunchers[0]),
298+
mock_crunch.call_args_list[0][0],
299+
)
300+
self.assertDictEqual(
301+
search_result.search_kwargs, mock_crunch.call_args_list[0][1]
302+
)
303+
304+
# chack that download() method is called for initial search result and the one added to it
305+
self.assertEqual(mock_download.call_count, 2)
306+
self.assertEqual(tmp_search_result[0], mock_download.call_args_list[0].args[0])
307+
self.assertEqual(
308+
search_iter_page_search_result_2[0], mock_download.call_args_list[1].args[0]
309+
)
310+
311+
# check that two paths are returned
312+
self.assertEqual(len(paths), 2)
313+
314+
# check that logs have been well raised
315+
self.assertIn("Searching other products from all pages", str(cm.output))
316+
self.assertIn(
317+
"Search on page #1 skipped since it was already requested", str(cm.output)
318+
)
319+
self.assertIn(
320+
"Iterate over pages: last products found on page 3", str(cm.output)
321+
)
322+
self.assertIn("Found 2 other result(s)", str(cm.output))
323+
self.assertIn("Apply the crunchers used on initial results", str(cm.output))
324+
self.assertIn("1 result(s) crunched", str(cm.output))
325+
self.assertIn("Downloading 2 product(s)", str(cm.output))
326+
327+
@mock.patch("eodag.api.product._product.EOProduct.download", autospec=True)
328+
@mock.patch("eodag.api.core.EODataAccessGateway.search_iter_page", autospec=True)
329+
def test_plugins_download_base_download_all_exhaust_after_search_all(
330+
self, mock_search_iter_page, mock_download
331+
):
332+
"""Download.download_all with "exhaust" parameter set to True must download
333+
only products in arguments if they are results of search_all()"""
334+
mock_search_iter_page.return_value = mock.ANY
335+
336+
# simulate a search result which is a result of search_all(), its "search kwargs" attribute is keep to None then
337+
search_result = SearchResult([self.product])
338+
339+
dag = EODataAccessGateway()
340+
341+
plugin = self.get_download_plugin(self.product)
342+
343+
with self.assertLogs(level="DEBUG") as cm:
344+
paths = plugin.download_all(search_result, dag, exhaust=True)
345+
346+
# check that search_iter_page() method is not called
347+
mock_search_iter_page.assert_not_called()
348+
349+
# chack that download() method is called only for the initial search result
350+
self.assertEqual(mock_download.call_count, 1)
351+
self.assertEqual(search_result[0], mock_download.call_args_list[0].args[0])
352+
353+
# check that two paths are returned
354+
self.assertEqual(len(paths), 1)
355+
356+
# check that logs have been well raised
357+
self.assertIn("Searching other products from all pages", str(cm.output))
358+
self.assertIn(
359+
"Products from all pages have already been searched", str(cm.output)
360+
)
361+
self.assertIn("Downloading 1 product(s)", str(cm.output))
362+
165363

166364
class TestDownloadPluginHttp(BaseDownloadPluginTest):
167365
def _download_response_archive(self, local_product_as_archive_path: str):

0 commit comments

Comments
 (0)