Merge pull request #444 from NREL/fix-pandas-deprecation-warnings-in-…

…tests

Fix pandas deprecation warnings in tests

docs/sphinx/source/changelog/pending.rst

@@ -14,6 +14,7 @@ Bug fixes
 * Fix :py:func:`~rdtools.normalization.energy_from_power` returns incorrect index for shifted hourly data (:issue:`370`, :pull:`437`)
 * Add warning to clearsky workflow when ``power_expected`` is passed by user (:pull:`439`)
 * Fix different results with Nan's and Zeros in power series (:issue:`313`, :pull:`442`)
+* Fix pandas deprecation warnings in tests (:pull:`444`)
 
 
 Requirements

rdtools/analysis_chains.py

@@ -619,17 +619,16 @@ def _call_clearsky_filter(filter_string):
                 warnings.warn(
                     "ad_hoc_filter contains NaN values; setting to False (excluding)"
                 )
-                ad_hoc_filter = ad_hoc_filter.fillna(False)
+                ad_hoc_filter.loc[ad_hoc_filter.isnull()] = False
 
             if not filter_components.index.equals(ad_hoc_filter.index):
                 warnings.warn(
                     "ad_hoc_filter index does not match index of other filters; missing "
                     "values will be set to True (kept). Align the index with the index "
                     "of the filter_components attribute to prevent this warning"
                 )
-                ad_hoc_filter = ad_hoc_filter.reindex(filter_components.index).fillna(
-                    True
-                )
+                ad_hoc_filter = ad_hoc_filter.reindex(filter_components.index)
+                ad_hoc_filter.loc[ad_hoc_filter.isnull()] = True
 
             filter_components["ad_hoc_filter"] = ad_hoc_filter
 
@@ -710,7 +709,7 @@ def _aggregated_filter(self, aggregated, case):
                 warnings.warn(
                     "aggregated ad_hoc_filter contains NaN values; setting to False (excluding)"
                 )
-                ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.fillna(False)
+                ad_hoc_filter_aggregated.loc[ad_hoc_filter_aggregated.isnull()] = False
 
             if not filter_components_aggregated.index.equals(
                 ad_hoc_filter_aggregated.index
@@ -723,7 +722,8 @@ def _aggregated_filter(self, aggregated, case):
                 )
                 ad_hoc_filter_aggregated = ad_hoc_filter_aggregated.reindex(
                     filter_components_aggregated.index
-                ).fillna(True)
+                )
+                ad_hoc_filter_aggregated.loc[ad_hoc_filter_aggregated.isnull()] = True
 
             filter_components_aggregated["ad_hoc_filter"] = ad_hoc_filter_aggregated
 

rdtools/availability.py

@@ -269,7 +269,7 @@ def _calc_loss_subsystem(self, low_threshold, relative_sizes,
         subsystem_fraction = relative_sizes / relative_sizes.sum()
         smallest_delta = (
             power_subsystem.le(low_threshold)
-            .replace(False, np.nan)
+            .replace(False, None)
             .multiply(subsystem_fraction)
             .min(axis=1)
             .astype(float)

rdtools/degradation.py

@@ -130,7 +130,9 @@ def degradation_classical_decomposition(energy_normalized,
     # Compute yearly rolling mean to isolate trend component using
     # moving average
     energy_ma = df['energy_normalized'].rolling('365d', center=True).mean()
-    has_full_year = (df['years'] >= df['years'][0] + 0.5) & (df['years'] <= df['years'][-1] - 0.5)
+    has_full_year = (df["years"] >= df["years"].iloc[0] + 0.5) & (
+        df["years"] <= df["years"].iloc[-1] - 0.5
+    )
     energy_ma[~has_full_year] = np.nan
     df['energy_ma'] = energy_ma
 

rdtools/test/analysis_chains_test.py

@@ -42,7 +42,7 @@ def degradation_trend(basic_parameters, cs_input):
     from degradation_test import DegradationTestCase
 
     rd = -0.05
-    input_freq = "H"
+    input_freq = "h"
     degradation_trend = DegradationTestCase.get_corr_energy(rd, input_freq)
     tz = cs_input["pvlib_location"].tz
     return degradation_trend.tz_localize(tz)
@@ -57,7 +57,7 @@ def sensor_parameters(basic_parameters, degradation_trend):
     basic_parameters["pv"] = power
     basic_parameters["poa_global"] = poa_global
     basic_parameters["temperature_ambient"] = temperature_ambient
-    basic_parameters["interp_freq"] = "H"
+    basic_parameters["interp_freq"] = "h"
     return basic_parameters
 
 
@@ -197,7 +197,7 @@ def test_interpolation(basic_parameters, degradation_trend):
     basic_parameters["temperature_cell"] = dummy_series
     basic_parameters["windspeed"] = dummy_series
     basic_parameters["power_expected"] = dummy_series
-    basic_parameters["interp_freq"] = "H"
+    basic_parameters["interp_freq"] = "h"
 
     rd_analysis = TrendAnalysis(**basic_parameters)
 
@@ -457,8 +457,8 @@ def test_filter_ad_hoc_warnings(workflow, sensor_parameters):
     assert components["ad_hoc_filter"].all()
 
     # warning about NaNs
-    ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index)
-    ad_hoc_filter.iloc[10] = np.nan
+    ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index, dtype="boolean")
+    ad_hoc_filter.iloc[10] = pd.NA
     rd_analysis.filter_params["ad_hoc_filter"] = ad_hoc_filter
     with pytest.warns(
         UserWarning, match="ad_hoc_filter contains NaN values; setting to False"
@@ -508,8 +508,10 @@ def test_aggregated_filter_ad_hoc_warnings(workflow, sensor_parameters):
     # disable all filters outside of CSI
     rd_analysis_2.filter_params = {"clearsky_filter": {"model": "csi"}}
     daily_ad_hoc_filter = pd.Series(True, index=sensor_parameters["pv"].index)
-    daily_ad_hoc_filter = daily_ad_hoc_filter.resample("1D").first().dropna(how="all")
-    daily_ad_hoc_filter.iloc[10] = np.nan
+    daily_ad_hoc_filter = (
+        daily_ad_hoc_filter.resample("1D").first().dropna(how="all").astype("boolean")
+    )
+    daily_ad_hoc_filter.iloc[10] = pd.NA
     rd_analysis_2.filter_params_aggregated["ad_hoc_filter"] = daily_ad_hoc_filter
     with pytest.warns(
         UserWarning, match="ad_hoc_filter contains NaN values; setting to False"

rdtools/test/degradation_test.py

@@ -53,20 +53,39 @@ def setUpClass(cls):
         # define module constants and parameters
 
         # All frequencies
-        cls.list_all_input_freq = ["MS", "M", "W", "D", "h", "min", "s", "Irregular_D"]
+        cls.list_all_input_freq = ["MS", "ME", "W", "D", "h", "min", "s", "Irregular_D"]
 
         # Allowed frequencies for degradation_ols
-        cls.list_ols_input_freq = ["MS", "M", "W", "D", "h", "min", "s", "Irregular_D"]
+        cls.list_ols_input_freq = ["MS", "ME", "W", "D", "h", "min", "s", "Irregular_D"]
 
         '''
         Allowed frequencies for degradation_classical_decomposition
         in principle CD works on higher frequency data but that makes the
         tests painfully slow
         '''
-        cls.list_CD_input_freq = ['MS', 'M', 'W', 'D']
+        cls.list_CD_input_freq = ["MS", "ME", "W", "D"]
 
         # Allowed frequencies for degradation_year_on_year
-        cls.list_YOY_input_freq = ['MS', 'M', 'W', 'D', 'Irregular_D']
+        cls.list_YOY_input_freq = ["MS", "ME", "W", "D", "Irregular_D"]
+
+        # ------------------------------------------------------------------------------------------------
+        # Allow pandas < 2.2.0 to use 'M' as an alias for MonthEnd
+        # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#deprecate-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets
+        # Check pandas version and set frequency alias
+        pandas_version = pd.__version__.split(".")
+        if int(pandas_version[0]) < 2 or (
+            int(pandas_version[0]) == 2 and int(pandas_version[1]) < 2
+        ):
+            for list in [
+                cls.list_all_input_freq,
+                cls.list_ols_input_freq,
+                cls.list_CD_input_freq,
+                cls.list_YOY_input_freq,
+            ]:
+                if "ME" in list:
+                    list.remove("ME")
+                    list.append(pd.tseries.offsets.MonthEnd())
+        # ------------------------------------------------------------------------------------------------
 
         cls.rd = -0.005
 
@@ -184,18 +203,27 @@ def test_usage_of_points(self):
         self.assertTrue((np.sum(rd_result[2]['usage_of_points'])) == 1462)
 
 
-@pytest.mark.parametrize('start,end,freq', [
-    ('2014-01-01', '2015-12-31', 'D'),   # no leap day
-    ('2015-01-01', '2016-12-31', 'D'),   # leap day included in index
-    ('2015-01-01', '2016-12-29', '7D'),  # leap day in period but not in index
-    ('2016-06-01', '2018-05-31', 'D'),   # leap year, but no leap day in period
-    #  ('2016-02-29', '2018-02-28', 'd'),   # starts on leap day (doesn't work)
-    ('2014-03-01', '2016-02-29', 'D'),   # ends on leap day
-    ('2015-01-01', '2016-12-31', 'M'),   # month end
-    ('2015-01-01', '2016-12-31', 'MS'),   # month start
-])
+@pytest.mark.parametrize(
+    "start,end,freq",
+    [
+        ("2014-01-01", "2015-12-31", "D"),  # no leap day
+        ("2015-01-01", "2016-12-31", "D"),  # leap day included in index
+        ("2015-01-01", "2016-12-29", "7D"),  # leap day in period but not in index
+        ("2016-06-01", "2018-05-31", "D"),  # leap year, but no leap day in period
+        #  ('2016-02-29', '2018-02-28', 'd'),   # starts on leap day (doesn't work)
+        ("2014-03-01", "2016-02-29", "D"),  # ends on leap day
+        ("2015-01-01", "2016-12-31", "ME"),  # month end
+        ("2015-01-01", "2016-12-31", "MS"),  # month start
+    ],
+)
 def test_yoy_two_years_error(start, end, freq):
-    # GH 339
+    # ----------------------------------------------------------------
+    # Allow pandas < 2.2.0 to use 'M' as an alias for MonthEnd
+    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#deprecate-aliases-m-q-y-etc-in-favour-of-me-qe-ye-etc-for-offsets
+    if freq == "ME":
+        freq = pd.tseries.offsets.MonthEnd()
+    # ----------------------------------------------------------------
+
     times = pd.date_range(start, end, freq=freq)
     series = pd.Series(1, index=times)
     # introduce NaN at the end to ensure that the 2 year requirement applies to

rdtools/test/energy_from_power_test.py

@@ -103,7 +103,7 @@ def test_energy_from_power_single_value_with_target():
     times = pd.date_range("2019-01-01", freq="15min", periods=1)
     power = pd.Series([100.0], index=times)
     expected_result = pd.Series([100.0], index=times, name="energy_Wh")
-    result = energy_from_power(power, target_frequency="H")
+    result = energy_from_power(power, target_frequency="h")
     pd.testing.assert_series_equal(result, expected_result)
 
 

rdtools/test/filtering_test.py

@@ -129,19 +129,22 @@ def generate_power_time_series_no_clipping():
 def generate_power_time_series_irregular_intervals():
     power_datetime_index = pd.Series(np.arange(1, 62))
     # Add datetime index to second series
-    time_range_1 = pd.date_range('2016-12-02T11:00:00.000Z',
-                                 '2017-06-06T07:00:00.000Z', freq='1T')
+    time_range_1 = pd.date_range(
+        "2016-12-02T11:00:00.000Z", "2017-06-06T07:00:00.000Z", freq="1min"
+    )
     power_datetime_index.index = pd.to_datetime(time_range_1[:61])
     power_datetime_index_2 = pd.Series(np.arange(100, 200))
-    time_range_2 = pd.date_range(power_datetime_index.index.max(),
-                                 '2017-06-06T07:00:00.000Z', freq='15T')
+    time_range_2 = pd.date_range(
+        power_datetime_index.index.max(), "2017-06-06T07:00:00.000Z", freq="15min"
+    )
     power_datetime_index_2.index = pd.to_datetime(time_range_2[:100])
     power_datetime_index_2 = power_datetime_index_2.iloc[1:]
     power_datetime_index = pd.concat([power_datetime_index,
                                       power_datetime_index_2])
     power_datetime_index_3 = pd.Series(list(reversed(np.arange(100, 200))))
-    time_range_3 = pd.date_range(power_datetime_index.index.max(),
-                                 '2017-06-06T07:00:00.000Z', freq='5T')
+    time_range_3 = pd.date_range(
+        power_datetime_index.index.max(), "2017-06-06T07:00:00.000Z", freq="5min"
+    )
     power_datetime_index_3.index = pd.to_datetime(time_range_3[:100])
     power_datetime_index_3 = power_datetime_index_3.iloc[1:]
     power_datetime_index = pd.concat([power_datetime_index,
@@ -157,8 +160,7 @@ def generate_power_time_series_one_min_intervals():
     power_datetime_index = pd.concat([power_datetime_index,
                                       power_datetime_index[::-1]])
     # Add datetime index to second series
-    time_range = pd.date_range('2016-12-02T11:00:00.000Z',
-                               '2017-06-06T07:00:00.000Z', freq='1T')
+    time_range = pd.date_range("2016-12-02T11:00:00.000Z", "2017-06-06T07:00:00.000Z", freq="1min")
     power_datetime_index.index = pd.to_datetime(time_range[:100])
     # Note: Power is expected to be Series object with a datetime index.
     return power_datetime_index