option to ignore first part of forecast in the model (#202)

* option to ignore first part of forecast in the model

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix

* correction for sun features

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix

* init

* add DA experiments

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add images

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: peterdudfield

experiments/india/006_da_only/bad.png

@@ -0,0 +1,37 @@
+## DA forecasts only
+
+The idea was to create a forecast for DA (day-ahead) only for Windnet.
+We hope this would bring down the DA MAE values.
+
+We do this by not forecasting the first X hours.
+
+Unfortunately, it doesnt not look like ignore X hours, make the DA forecast better.
+
+## Experiments
+
+1. Baseline - [here](https://wandb.ai/openclimatefix/india/runs/miszfep5)
+2. Ignore first 6 hours - [here](https://wandb.ai/openclimatefix/india/runs/uosk0qug)
+3. Ignore first 12 hours - [here](https://wandb.ai/openclimatefix/india/runs/s9cnn4ei)
+
+## Results
+
+| Timestep | all MAE % | 6 MAE % | 12 MAE % |
+| --- | --- |---------|---------|
+| 0-0 minutes | nan | nan     | nan     |
+| 15-15 minutes | nan | nan     | nan     |
+| 30-45 minutes | 0.065 | nan     | nan     |
+| 45-60 minutes | 0.066 | nan     | nan     |
+| 60-120 minutes | 0.063 | nan     | nan     |
+| 120-240 minutes | 0.063 | nan     | nan     |
+| 240-360 minutes | 0.064 | nan     | nan     |
+| 360-480 minutes | 0.065 | 0.068   | nan     |
+| 480-720 minutes | 0.067 | 0.065   | nan     |
+| 720-1440 minutes | 0.068 | 0.065   | 0.065   |
+| 1440-2880 minutes | 0.071 | 0.071   | 0.071   |
+
+![](mae_steps.png "mae_steps")
+
+Here's two examples from the 6 hour ignore model, one that forecated it well, one that didnt
+
+![](bad.png "bad")
+![](good.png "good")

experiments/india/006_da_only/da_only.md

@@ -245,7 +245,7 @@ def save_pretrained(
 
 
 class BaseModel(pl.LightningModule, PVNetModelHubMixin):
-    """Abtstract base class for PVNet submodels"""
+    """Abstract base class for PVNet submodels"""
 
     def __init__(
         self,
@@ -257,6 +257,7 @@ def __init__(
         interval_minutes: int = 30,
         timestep_intervals_to_plot: Optional[list[int]] = None,
         use_weighted_loss: bool = False,
+        forecast_minutes_ignore: Optional[int] = 0,
     ):
         """Abtstract base class for PVNet submodels.
 
@@ -270,6 +271,8 @@ def __init__(
             interval_minutes: The interval in minutes between each timestep in the data
             timestep_intervals_to_plot: Intervals, in timesteps, to plot during training
             use_weighted_loss: Whether to use a weighted loss function
+            forecast_minutes_ignore: Number of forecast minutes to ignore when calculating losses.
+                For example if set to 60, the model doesnt predict the first 60 minutes
         """
         super().__init__()
 
@@ -292,10 +295,12 @@ def __init__(
         self.forecast_minutes = forecast_minutes
         self.output_quantiles = output_quantiles
         self.interval_minutes = interval_minutes
+        self.forecast_minutes_ignore = forecast_minutes_ignore
 
         # Number of timestemps for 30 minutely data
         self.history_len = history_minutes // interval_minutes
-        self.forecast_len = forecast_minutes // interval_minutes
+        self.forecast_len = (forecast_minutes - forecast_minutes_ignore) // interval_minutes
+        self.forecast_len_ignore = forecast_minutes_ignore // interval_minutes
 
         self.weighted_losses = WeightedLosses(forecast_length=self.forecast_len)
 
@@ -334,7 +339,7 @@ def _quantiles_to_prediction(self, y_quantiles):
         y_median = y_quantiles[..., idx]
         return y_median
 
-    def _calculate_qauntile_loss(self, y_quantiles, y):
+    def _calculate_quantile_loss(self, y_quantiles, y):
         """Calculate quantile loss.
 
         Note:
@@ -366,7 +371,7 @@ def _calculate_common_losses(self, y, y_hat):
         losses = {}
 
         if self.use_quantile_regression:
-            losses["quantile_loss"] = self._calculate_qauntile_loss(y_hat, y)
+            losses["quantile_loss"] = self._calculate_quantile_loss(y_hat, y)
             y_hat = self._quantiles_to_prediction(y_hat)
 
         # calculate mse, mae

experiments/india/006_da_only/good.png

@@ -71,6 +71,7 @@ def __init__(
         timestep_intervals_to_plot: Optional[list[int]] = None,
         adapt_batches: Optional[bool] = False,
         use_weighted_loss: Optional[bool] = False,
+        forecast_minutes_ignore: Optional[int] = 0,
     ):
         """Neural network which combines information from different sources.
 
@@ -131,6 +132,8 @@ def __init__(
                 the model to use. This allows us to overprepare batches and slice from them for the
                 data we need for a model run.
             use_weighted_loss: Whether to use a weighted loss function
+            forecast_minutes_ignore: Number of forecast minutes to ignore when calculating losses.
+                For example if set to 60, the model doesnt predict the first 60 minutes
         """
 
         self.include_gsp_yield_history = include_gsp_yield_history
@@ -154,6 +157,7 @@ def __init__(
             interval_minutes=interval_minutes,
             timestep_intervals_to_plot=timestep_intervals_to_plot,
             use_weighted_loss=use_weighted_loss,
+            forecast_minutes_ignore=forecast_minutes_ignore,
         )
 
         # Number of features expected by the output_network
@@ -271,7 +275,8 @@ def __init__(
 
         if self.include_sun:
             self.sun_fc1 = nn.Linear(
-                in_features=2 * (self.forecast_len + self.history_len + 1),
+                in_features=2
+                * (self.forecast_len + self.forecast_len_ignore + self.history_len + 1),
                 out_features=16,
             )
 

experiments/india/006_da_only/mae_steps.png

@@ -303,3 +303,12 @@ def multimodal_weighted_quantile_model(multimodal_model_kwargs):
         output_quantiles=[0.1, 0.5, 0.9], **multimodal_model_kwargs, use_weighted_loss=True
     )
     return model
+
+
+@pytest.fixture()
+def multimodal_quantile_model_ignore_minutes(multimodal_model_kwargs):
+    """Only forecsat second half of the 8 hours"""
+    model = Model(
+        output_quantiles=[0.1, 0.5, 0.9], **multimodal_model_kwargs, forecast_minutes_ignore=240
+    )
+    return model

pvnet/models/base_model.py

@@ -51,3 +51,14 @@ def test_weighted_quantile_model_backward(multimodal_weighted_quantile_model, sa
 
     # Backwards on sum drives sum to zero
     y_quantiles.sum().backward()
+
+
+def test_weighted_quantile_model_forward(multimodal_quantile_model_ignore_minutes, sample_batch):
+    y_quantiles = multimodal_quantile_model_ignore_minutes(sample_batch)
+
+    # check output is the correct shape
+    # batch size=2, forecast_len=8, num_quantiles=3
+    assert tuple(y_quantiles.shape) == (2, 8, 3), y_quantiles.shape
+
+    # Backwards on sum drives sum to zero
+    y_quantiles.sum().backward()