diff --git a/laika/astro_dog.py b/laika/astro_dog.py index 1d18ebf..9a56a19 100644 --- a/laika/astro_dog.py +++ b/laika/astro_dog.py @@ -1,3 +1,4 @@ +import logging import os from collections import defaultdict from concurrent.futures import ThreadPoolExecutor @@ -213,14 +214,21 @@ def download_parse_orbit(self, gps_time: GPSTime, skip_before_epoch=None) -> Map time_steps = [gps_time - SECS_IN_DAY, gps_time, gps_time + SECS_IN_DAY] with ThreadPoolExecutor() as executor: futures = [executor.submit(download_orbits_gps, t, self.cache_dir, self.valid_ephem_types) for t in time_steps] - files = [self.fetch_count(f.result()) for f in futures if f.result()] if futures else [] + files = [] + for t, future in zip(time_steps, futures): + try: + result = future.result() + if result: + files.append(self.fetch_count(result)) + except Exception as e: + logging.warning("Failed to download orbit for %s: %s", t.as_datetime(), e) ephems = parse_sp3_orbits(files, self.valid_const, skip_before_epoch) return ephems def get_orbit_data(self, time: GPSTime): ephems_sp3 = self.download_parse_orbit(time) if sum([len(v) for v in ephems_sp3.values()]) < 5: - raise RuntimeError(f'No orbit data found. For Time {time.as_datetime()} constellations {self.valid_const} valid ephem types {self.valid_ephem_types}') + raise RuntimeError(f'No orbit data found. For Time {time.as_datetime()} constellations {self.valid_const} valid ephem types {self.valid_ephem_types}. Check logs above for download failure details.') self.add_ephem_fetched_time(ephems_sp3, self.orbit_fetched_times) self.add_orbits(ephems_sp3) diff --git a/laika/downloader.py b/laika/downloader.py index 3432a05..5966416 100644 --- a/laika/downloader.py +++ b/laika/downloader.py @@ -43,13 +43,15 @@ def wrapped(url_bases, *args, **kwargs): return f(url_bases, *args, **kwargs) # not a string, must be a list of url_bases + errors = [] for url_base in (url for url in url_bases if url): try: return f(url_base, *args, **kwargs) except DownloadFailed as e: - logging.warning(e) + errors.append(str(e)) + logging.warning("Download from %s failed: %s", url_base, e) # none of them succeeded - raise DownloadFailed("Multiple URL failures attempting to pull file(s)") + raise DownloadFailed("Multiple URL failures attempting to pull file(s):\n" + "\n".join(errors)) return wrapped @@ -202,7 +204,7 @@ def https_download_file(url): crl.close() if response != 200: - raise DownloadFailed('HTTPS error ' + str(response)) + raise DownloadFailed(f'HTTPS error {response} downloading {url}') return buf.getvalue() @@ -218,7 +220,7 @@ def ftp_download_file(url): ftp.retrbinary('RETR ' + parsed.path, buf.write) return buf.getvalue() except ftplib.all_errors as e: - raise DownloadFailed(e) + raise DownloadFailed(f'FTP error downloading {url}: {e}') @retryable @@ -243,13 +245,18 @@ def download_file(url_base, folder_path, filename_zipped): def download_and_cache_file_return_first_success(url_bases, folder_and_file_names, cache_dir, compression='', overwrite=False, raise_error=False): last_error = None + errors = [] for folder_path, filename in folder_and_file_names: try: file = download_and_cache_file(url_bases, folder_path, cache_dir, filename, compression, overwrite) return file except DownloadFailed as e: + logging.warning("Download attempt failed for %s%s: %s", folder_path, filename, e) + errors.append(str(e)) last_error = e + if errors: + logging.warning("All download attempts failed:\n%s", "\n".join(errors)) if last_error and raise_error: raise last_error @@ -269,12 +276,13 @@ def download_and_cache_file(url_base, folder_path: str, cache_dir: str, filename if not os.path.isfile(filepath) or overwrite: try: data_zipped = download_file(url_base, folder_path, filename_zipped) - except (DownloadFailed, pycurl.error, TimeoutError): + except (DownloadFailed, pycurl.error, TimeoutError) as e: unix_time = time.time() os.makedirs(folder_path_abs, exist_ok=True) with atomic_write(filepath_attempt, mode='w', overwrite=True) as wf: wf.write(str(unix_time)) - raise DownloadFailed(f"Could not download {folder_path + filename_zipped} from {url_base}") + logging.warning("Failed to download %s from %s: %s", folder_path + filename_zipped, url_base, e) + raise DownloadFailed(f"Could not download {folder_path + filename_zipped} from {url_base}: {e}") os.makedirs(folder_path_abs, exist_ok=True) ephem_bytes = hatanaka.decompress(data_zipped) @@ -411,6 +419,6 @@ def download_cors_station(time, station_name, cache_dir): try: filepath = download_and_cache_file(url_bases, folder_path, cache_dir+'cors_obs/', filename, compression='.gz') return filepath - except DownloadFailed: - logging.warning("File not downloaded, check availability on server.") + except DownloadFailed as e: + logging.warning("File not downloaded for station %s: %s", station_name, e) return None