Wrapper for clearsky filter combining csi and pvlib models (#412)

* add wrapper clearsky_filter

* rename filter to model

* update analysis_chains with  clearsky filter wrapper

* update clearsky tests

* update default filter params

* add pending changelog

* update init.py

* update api.rst

* add test for clearsky_filter function

* remove experimental warning for logic_clip_filter test

* re-run notebook

* update configparser deprecated arguments python 3.12

* Use csi clearsky filter in a number of tests

* Adjust how attributes are removed for error testing

* fix linting in analysis_chains_test

* update numexpr to 2.10

* update h5py requirements to 3.11

* Revert "re-run notebook"

This reverts commit 751cfaa.

* update notebook requirements

* save degradation notebook without logic filter warning

* availability notebook avoid double plotting figures

* re-run TrendAnalysis notebook

* Revert "re-run TrendAnalysis notebook"

This reverts commit 3e91ada.

* restart kernel and re-run TrendAnalysis notebook

* update change log

* add line to change log

* further changelog fixes

---------

Co-authored-by: Michael Deceglie <Michael.Deceglie@nrel.gov>

Author: martin-springer

docs/TrendAnalysis_example_pvdaq4.ipynb

@@ -10,18 +10,18 @@ decorator==4.3.0
 defusedxml==0.7.1
 entrypoints==0.2.3
 html5lib==1.0.1
-ipykernel==4.8.2
-ipython==8.10.0
+ipykernel==6.29.4
+ipython==8.23.0
 ipython-genutils==0.2.0
 ipywidgets==7.3.0
 jedi==0.16.0
 Jinja2==3.0.0
 jsonschema==2.6.0
 jupyter==1.0.0
-jupyter-client==6.1.7
-jupyter-console==6.4.0
-jupyter-core==4.11.2
-jupyterlab-pygments==0.2.2
+jupyter-client==8.6.1
+jupyter-console==6.6.3
+jupyter-core==5.7.2
+jupyterlab-pygments==0.3.0
 lxml==4.9.1
 MarkupSafe==2.0.0
 mistune==2.0.3
@@ -30,17 +30,17 @@ nbconvert==7.0.0
 nbformat==5.1.0
 nest-asyncio==1.5.5
 notebook==6.4.12
-numexpr==2.8.0
+numexpr==2.10.0
 pandocfilters==1.4.2
 parso==0.5.2
 pexpect==4.6.0
 pickleshare==0.7.5
 prometheus-client==0.3.0
-prompt-toolkit==3.0.30
+prompt-toolkit==3.0.43
 ptyprocess==0.6.0
 pycparser==2.20
 Pygments==2.15.0
-pyzmq==22.2.1
+pyzmq==26.0.2
 qtconsole==4.3.1
 Send2Trash==1.8.0
 simplegeneric==0.8.1
@@ -49,7 +49,7 @@ terminado==0.8.3
 testpath==0.3.1
 tinycss2==1.1.1
 tornado==6.3.3
-traitlets==5.0.0
+traitlets==5.14.3
 wcwidth==0.1.7
 webencodings==0.5.1
 widgetsnbextension==3.3.0

docs/degradation_and_soiling_example_pvdaq_4.ipynb

@@ -9,14 +9,14 @@ Submodules
 ==========
 
 RdTools is organized into submodules focused on different parts of the data
-analysis workflow.  
+analysis workflow.
 
 .. autosummary::
    :toctree: generated/
 
    analysis_chains
    degradation
-   soiling   
+   soiling
    availability
    filtering
    normalization
@@ -42,8 +42,8 @@ Object-oriented end-to-end analysis
    analysis_chains.TrendAnalysis.plot_soiling_monte_carlo
    analysis_chains.TrendAnalysis.plot_pv_vs_irradiance
    analysis_chains.TrendAnalysis.plot_degradation_timeseries
-   
-   
+
+
 Degradation
 ===========
 
@@ -86,7 +86,7 @@ System Availability
 
 .. autosummary::
    :toctree: generated/
-   
+
    AvailabilityAnalysis
    AvailabilityAnalysis.run
    AvailabilityAnalysis.plot
@@ -100,12 +100,14 @@ Filtering
 
 .. autosummary::
    :toctree: generated/
-    
+
    clip_filter
    quantile_clip_filter
    logic_clip_filter
    xgboost_clip_filter
+   clearsky_filter
    csi_filter
+   pvlib_clearsky_filter
    poa_filter
    tcell_filter
    normalized_filter

docs/notebook_requirements.txt

@@ -1,5 +1,6 @@
 RdTools Change Log
 ==================
+.. include:: changelog/pending.rst
 .. include:: changelog/v2.2.0-beta.2.rst
 .. include:: changelog/v2.2.0-beta.1.rst
 .. include:: changelog/v2.1.8.rst

docs/sphinx/source/api.rst

@@ -0,0 +1,27 @@
+*******
+pending
+*******
+
+Breaking changes
+------------
+These changes have the potential to change answers in existing scripts
+when compared with older versions of RdTools
+
+* Use the pvlib method for clear sky detection by default in :py:func:`~rdtools.analysis_chains.TrendAnalysis` (:pull:`412`)
+
+Enhancements
+------------
+* Added a new wrapper function for clearsky filters (:pull:`412`)
+
+Bug fixes
+---------
+* tbd
+
+Requirements
+------------
+* Specified versions in ``requirements.txt`` and ``docs/notebook_requirements.txt`` have been updated (:pull:`412`)
+
+Contributors
+------------
+* Martin Springer (:ghuser:`martin-springer`)
+* Michael Deceglie (:ghuser:`mdeceglie`)

docs/sphinx/source/changelog.rst

@@ -9,7 +9,9 @@
 from rdtools.degradation import degradation_year_on_year
 from rdtools.aggregation import aggregation_insol
 from rdtools.clearsky_temperature import get_clearsky_tamb
+from rdtools.filtering import clearsky_filter
 from rdtools.filtering import csi_filter
+from rdtools.filtering import pvlib_clearsky_filter
 from rdtools.filtering import poa_filter
 from rdtools.filtering import tcell_filter
 from rdtools.filtering import clip_filter

docs/sphinx/source/changelog/pending.rst

@@ -153,7 +153,7 @@ def __init__(
             "poa_filter": {},
             "tcell_filter": {},
             "clip_filter": {},
-            "pvlib_clearsky_filter": {},
+            "clearsky_filter": {},
             "ad_hoc_filter": None,  # use this to include an explict filter
         }
         self.filter_params_aggregated = {"ad_hoc_filter": None}
@@ -461,7 +461,7 @@ def _call_clearsky_filter(filter_string):
                     "Both poa_global and poa_global_clearsky must be available to "
                     f"do clearsky filtering with {filter_string}"
                 )
-            f = filtering.pvlib_clearsky_filter(
+            f = filtering.clearsky_filter(
                 self.poa_global,
                 self.poa_global_clearsky,
                 **self.filter_params[filter_string],
@@ -529,12 +529,12 @@ def _call_clearsky_filter(filter_string):
             filter_components["hour_angle_filter"] = f
 
         if case == "clearsky":
-            filter_components["pvlib_clearsky_filter"] = _call_clearsky_filter(
-                "pvlib_clearsky_filter"
+            filter_components["clearsky_filter"] = _call_clearsky_filter(
+                "clearsky_filter"
             )
-        if "sensor_pvlib_clearsky_filter" in self.filter_params:
-            filter_components["sensor_pvlib_clearsky_filter"] = _call_clearsky_filter(
-                "sensor_pvlib_clearsky_filter"
+        if "sensor_clearsky_filter" in self.filter_params:
+            filter_components["sensor_clearsky_filter"] = _call_clearsky_filter(
+                "sensor_clearsky_filter"
             )
 
         # note: the previous implementation using the & operator treated NaN
@@ -803,14 +803,14 @@ def _sensor_preprocess(self):
                 "poa_global must be available to perform _sensor_preprocess"
             )
 
-        if "sensor_pvlib_clearsky_filter" in self.filter_params:
+        if "sensor_clearsky_filter" in self.filter_params:
             try:
                 if self.poa_global_clearsky is None:
                     self._calc_clearsky_poa(model="isotropic")
             except AttributeError:
                 raise AttributeError(
                     "No poa_global_clearsky. 'set_clearsky' must be run "
-                    + "to allow filter_params['sensor_pvlib_clearsky_filter']. "
+                    + "to allow filter_params['sensor_clearsky_filter']. "
                 )
         if self.power_expected is None:
             # Thermal details required if power_expected is not manually set.

docs/system_availability_example.ipynb

@@ -103,6 +103,43 @@ def tcell_filter(temperature_cell, temperature_cell_low=-50, temperature_cell_hi
     )
 
 
+def clearsky_filter(poa_global_measured, poa_global_clearsky, model='pvlib', **kwargs):
+    """
+    Wrapper function for running either the CSI or pvlib clearsky filter.
+
+    Parameters
+    ----------
+    poa_global_measured : pandas.Series
+        Plane of array irradiance based on measurments
+    poa_global_clearsky : pandas.Series
+        Plane of array irradiance based on a clear sky model
+    model : str, default 'pvlib'
+        Clearsky filter model to be applied. Can be 'pvlib' or 'csi'.
+    kwargs :
+        Additional clearsky filter args, specific to the filter being
+        used. Keyword must be passed with value.
+
+    Returns
+    -------
+    pandas.Series
+        Boolean Series of whether or not the given time is clear
+        based on the selected filter.
+
+    See Also
+    --------
+    csi_filter : Filtering based on clear-sky index (csi).
+    pvlib_clearsky_filter : Filtering based on pvlib's clearsky model.
+    """
+
+    if model == "csi":
+        clearsky_mask = csi_filter(poa_global_measured, poa_global_clearsky, **kwargs)
+    elif model == "pvlib":
+        clearsky_mask = pvlib_clearsky_filter(poa_global_measured, poa_global_clearsky, **kwargs)
+    else:
+        raise ValueError("Clearsky filter must be 'pvlib' or 'csi'.")
+    return clearsky_mask
+
+
 def csi_filter(poa_global_measured, poa_global_clearsky, threshold=0.15):
     """
     Filtering based on clear-sky index (csi)