From 577d4ee1aadf43e9f417f9c372a21326df854b22 Mon Sep 17 00:00:00 2001 From: Lilli Freischem Date: Fri, 17 May 2024 10:28:51 +0000 Subject: [PATCH] normalisation --- environments/environment_iti.yaml | 5 +- notebooks/2.0-test-normalization.ipynb | 517 ++++++++++++++++++++++- rs_tools/_src/preprocessing/normalize.py | 1 - 3 files changed, 519 insertions(+), 4 deletions(-) diff --git a/environments/environment_iti.yaml b/environments/environment_iti.yaml index ea3a68c..c0febdb 100644 --- a/environments/environment_iti.yaml +++ b/environments/environment_iti.yaml @@ -1,6 +1,8 @@ name: rs_tools_iti channels: - conda-forge + - pytorch + - nvidia dependencies: - python=3.11 # Standard Libraries @@ -24,7 +26,8 @@ dependencies: - zarr - h5netcdf - pyhdf - - nvidia/label/cuda-12.2.0::cuda + # - nvidia/label/cuda-12.2.0::cuda + - pytorch-cuda # - h5py # GUI - ipywidgets diff --git a/notebooks/2.0-test-normalization.ipynb b/notebooks/2.0-test-normalization.ipynb index cd88817..47cddab 100644 --- a/notebooks/2.0-test-normalization.ipynb +++ b/notebooks/2.0-test-normalization.ipynb @@ -25,6 +25,80 @@ "%autoreload 2" ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "\n", + "norm_dir = '/home/freischem/outputs/miniset/20240517-0928/normalization'\n", + "goes_norm = xr.open_dataset(f'{norm_dir}/goes_norm.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([134.38202 , 65.178566 , 37.20662 , 2.272993 ,\n", + " 5.744725 , 1.5239868 , 0.77519065, 2.815666 ,\n", + " 7.9387918 , 13.896899 , 47.947056 , 43.477985 ,\n", + " 82.78253 , 93.38069 , 100.64417 , 88.33139 ],\n", + " dtype=float32),\n", + " array([83.57160446, 70.23925393, 47.01136956, 6.13335517, 6.55724009,\n", + " 1.74397223, 0.245557 , 0.69149961, 1.84830359, 3.10537042,\n", + " 11.73538443, 8.68892694, 18.23192365, 20.14717129, 20.09120789,\n", + " 14.56906235]))" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "goes_norm['mean'].values, goes_norm['std'].values" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "msg_norm = xr.open_dataset(f'{norm_dir}/msg_norm.nc')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(array([ 1.695779 , 0.6518643, 52.94932 , 45.511276 , 86.90486 ,\n", + " 97.99122 , 80.530754 , 2.6322215, 2.830637 , 3.2238169,\n", + " 14.109417 ], dtype=float32),\n", + " array([ 1.65503616, 0.23466894, 15.14200648, 10.67773555, 22.5324939 ,\n", + " 23.94238188, 14.97517086, 2.73353005, 3.34456361, 0.8772509 ,\n", + " 3.74212206]))" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "msg_norm['mean'].values, msg_norm['std'].values" + ] + }, { "cell_type": "code", "execution_count": 3, @@ -522,9 +596,448 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 125, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 78MB\n",
+       "Dimensions:          (y: 1289, x: 891, band: 11, time: 1, band_wavelength: 11)\n",
+       "Coordinates:\n",
+       "  * y                (y) float64 10kB 0.0 3e+03 ... 3.862e+06 3.865e+06\n",
+       "  * x                (x) float64 7kB -4.264e+06 -4.261e+06 ... -1.593e+06\n",
+       "    cloud_mask       (y, x) float64 9MB ...\n",
+       "    latitude         (y, x) float64 9MB ...\n",
+       "    longitude        (y, x) float64 9MB ...\n",
+       "  * band             (band) <U6 264B 'IR_016' 'IR_039' ... 'WV_062' 'WV_073'\n",
+       "  * time             (time) <U19 76B '2020-10-21 14:00:00'\n",
+       "  * band_wavelength  (band_wavelength) float64 88B 1.64 3.92 8.7 ... 6.25 7.35\n",
+       "Data variables:\n",
+       "    Rad              (band, y, x) float32 51MB ...\n",
+       "Attributes:\n",
+       "    calibration:         radiance\n",
+       "    standard_name:       toa_outgoing_radiance_per_unit_wavenumber\n",
+       "    platform_name:       Meteosat-11\n",
+       "    sensor:              seviri\n",
+       "    units:               mW m-2 sr-1 (cm-1)-1\n",
+       "    orbital_parameters:  {"projection_longitude": 0.0, "projection_latitude":...
" + ], + "text/plain": [ + " Size: 78MB\n", + "Dimensions: (y: 1289, x: 891, band: 11, time: 1, band_wavelength: 11)\n", + "Coordinates:\n", + " * y (y) float64 10kB 0.0 3e+03 ... 3.862e+06 3.865e+06\n", + " * x (x) float64 7kB -4.264e+06 -4.261e+06 ... -1.593e+06\n", + " cloud_mask (y, x) float64 9MB ...\n", + " latitude (y, x) float64 9MB ...\n", + " longitude (y, x) float64 9MB ...\n", + " * band (band) xr.Dataset: return ds.mean(spatial_variables) -# TODO: Check this function def normalize( files: List[str], temporal_variables: List[str]=["time"],