Fix pandas v1.4 breaks the IIASA database API (#621)

pyam/core.py

@@ -1703,6 +1703,7 @@ def filter(self, keep=True, inplace=False, **kwargs):
         if len(idx) == 0:
             logger.warning("Filtered IamDataFrame is empty!")
         ret.meta = ret.meta.loc[idx]
+        ret.meta.index = ret.meta.index.remove_unused_levels()
         ret._set_attributes()
         if not inplace:
             return ret

pyam/iiasa.py

@@ -237,7 +237,7 @@ def _query_index(self, default=True, meta=False):
         _check_response(r)
 
         # cast response to dataframe and return
-        return pd.read_json(r.content, orient="records")
+        return pd.read_json(r.text, orient="records")
 
     @property
     @lru_cache()
@@ -247,7 +247,7 @@ def meta_columns(self):
         headers = {"Authorization": "Bearer {}".format(self._token)}
         r = requests.get(url, headers=headers)
         _check_response(r)
-        return pd.read_json(r.content, orient="records")["name"]
+        return pd.read_json(r.text, orient="records")["name"]
 
     def meta(self, default=True, **kwargs):
         """Return categories and indicators (meta) of scenarios
@@ -311,7 +311,7 @@ def variables(self):
         headers = {"Authorization": "Bearer {}".format(self._token)}
         r = requests.get(url, headers=headers)
         _check_response(r)
-        df = pd.read_json(r.content, orient="records")
+        df = pd.read_json(r.text, orient="records")
         return pd.Series(df["variable"].unique(), name="variable")
 
     @lru_cache()
@@ -330,7 +330,7 @@ def regions(self, include_synonyms=False):
         params = {"includeSynonyms": include_synonyms}
         r = requests.get(url, headers=headers, params=params)
         _check_response(r)
-        return self.convert_regions_payload(r.content, include_synonyms)
+        return self.convert_regions_payload(r.text, include_synonyms)
 
     @staticmethod
     def convert_regions_payload(response, include_synonyms):
@@ -485,8 +485,8 @@ def query(self, default=True, meta=True, **kwargs):
             value=float,
             version=int,
         )
-        data = pd.read_json(r.content, orient="records", dtype=dtype)
-        logger.debug(f"Response: {len(r.content)} bytes, {len(data)} records")
+        data = pd.read_json(r.text, orient="records", dtype=dtype)
+        logger.debug(f"Response: {len(r.text)} bytes, {len(data)} records")
         cols = IAMC_IDX + ["year", "value", "subannual", "version"]
         # keep only known columns or init empty df
         data = pd.DataFrame(data=data, columns=cols)

setup.cfg

@@ -29,7 +29,7 @@ install_requires =
     numpy >= 1.19.0
     requests
     openpyxl
-    pandas >= 1.1.1, < 1.4
+    pandas >= 1.1.1
     pint <= 0.17
     PyYAML
     matplotlib >= 3.2.0
@@ -48,9 +48,9 @@ setup_requires =
 tests =
     coverage
     coveralls
-    pytest < 6.0.0
+    pytest
     pytest-cov
-    pytest-mpl < 0.12
+    pytest-mpl
 optional_plotting =
     plotly
 optional_io_formats =

tests/test_admin.py

@@ -2,8 +2,8 @@
 to print configuration information of the loaded package(s) to users!"""
 import sys
 
-from test_plotting import MPL_KWARGS
-from conftest import IIASA_UNAVAILABLE
+from .test_plotting import MPL_KWARGS
+from .conftest import IIASA_UNAVAILABLE
 
 
 def test_config(capsys):

tests/test_cast_to_iamc.py

@@ -2,8 +2,6 @@
 import pandas as pd
 from pyam import IamDataFrame, compare
 
-from conftest import TEST_DTS
-
 
 # when making any updates to this file,
 # please also update the `data_table_formats` tutorial notebook!

tests/test_feature_aggregate.py

@@ -5,7 +5,8 @@
 import pandas as pd
 from pyam import check_aggregate, IamDataFrame, IAMC_IDX
 from pyam.testing import assert_iamframe_equal
-from conftest import TEST_YEARS, DTS_MAPPING
+
+from .conftest import DTS_MAPPING
 
 LONG_IDX = IAMC_IDX + ["year"]
 

tests/test_feature_append_rename.py

@@ -7,7 +7,7 @@
 
 from pyam import IamDataFrame, META_IDX, IAMC_IDX, compare
 
-from conftest import TEST_DTS, META_COLS
+from .conftest import META_COLS
 
 
 RENAME_DF = IamDataFrame(

tests/test_feature_growth_rate.py

@@ -6,7 +6,7 @@
 from pyam.timeseries import growth_rate
 import pytest
 
-from conftest import META_DF
+from .conftest import META_DF
 
 
 EXP_DF = IamDataFrame(

tests/test_iiasa.py

@@ -8,7 +8,8 @@
 
 from pyam import IamDataFrame, iiasa, read_iiasa, META_IDX
 from pyam.testing import assert_iamframe_equal
-from conftest import IIASA_UNAVAILABLE, META_COLS, TEST_API, TEST_API_NAME
+
+from .conftest import IIASA_UNAVAILABLE, META_COLS, TEST_API, TEST_API_NAME
 
 if IIASA_UNAVAILABLE:
     pytest.skip("IIASA database API unavailable", allow_module_level=True)

tests/test_io.py

@@ -7,7 +7,7 @@
 from pyam.utils import META_IDX
 from pyam.testing import assert_iamframe_equal
 
-from conftest import TEST_DATA_DIR, META_DF
+from .conftest import TEST_DATA_DIR, META_DF
 
 FILTER_ARGS = dict(scenario="scen_a")
 

tests/test_plotting.py

@@ -11,7 +11,7 @@
 
 import pyam
 from pyam import plotting, run_control, reset_rc_defaults
-from conftest import IMAGE_BASELINE_DIR
+from .conftest import IMAGE_BASELINE_DIR
 
 
 logger = logging.getLogger(__name__)

tests/test_run_control.py

@@ -3,7 +3,7 @@
 
 from pyam import IamDataFrame, run_control
 
-from conftest import TEST_DATA_DIR, TEST_DF
+from .conftest import TEST_DATA_DIR, TEST_DF
 
 
 def test_exec():

tests/test_time.py

@@ -4,7 +4,7 @@
 from pyam import IamDataFrame
 from pyam.testing import assert_iamframe_equal
 
-from conftest import TEST_DTS, TEST_TIME_STR, TEST_TIME_STR_HR
+from .conftest import TEST_DTS, TEST_TIME_STR, TEST_TIME_STR_HR
 
 
 def get_subannual_df(date1, date2):

tests/test_tutorials.py

@@ -4,7 +4,7 @@
 import sys
 import pytest
 
-from conftest import here, IIASA_UNAVAILABLE
+from .conftest import here, IIASA_UNAVAILABLE
 
 try:
     import nbformat