From 4ecacb7b7358f66ef07e5712aea39f1f1360262b Mon Sep 17 00:00:00 2001 From: Will Usher Date: Tue, 11 Jul 2023 15:30:54 +0200 Subject: [PATCH 1/5] Added fix for trade config with new read_file function --- src/osemosys2iamc/resultify.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/osemosys2iamc/resultify.py b/src/osemosys2iamc/resultify.py index c1b71e5..718cd8b 100644 --- a/src/osemosys2iamc/resultify.py +++ b/src/osemosys2iamc/resultify.py @@ -52,7 +52,7 @@ def iso_to_country( format_regex = r"^iso[23]_([1-9]\d*|start|end)$" # Verifies that given format is the expected format; Raises an error if expectation not met - if re.search(format_regex, iso_format) != None: + if re.search(format_regex, iso_format) is not None: iso_type, abbr_loc = iso_format[3:].split("_") @@ -75,14 +75,14 @@ def iso_to_country( """ Checks every technology/fuel name for a valid code. If found, adds that country to the list for that tech/fuel name, otherwise - adds an empty string for that tech/fuel. A position exceeeding the + adds an empty string for that tech/fuel. A position exceeding the length of the name, adds an empty string instead """ for i in index: if re.search(region_regex, i.upper()) != None: code = re.search(region_regex, i.upper()).groups()[0] if code in country_dict: - countries_list.append(country_dict[code].name.upper()) + countries_list.append(country_dict[code].name) else: countries_list.append("") no_country_extracted.append(i) @@ -431,7 +431,7 @@ def main(config: Dict, inputs_path: str, results_path: str) -> pyam.IamDataFrame try: for result in config["results"]: - if type(result["osemosys_param"]) == str: + if isinstance(result["osemosys_param"], str): results = read_file( results_path, result["osemosys_param"], config["region"] ) @@ -470,15 +470,24 @@ def main(config: Dict, inputs_path: str, results_path: str) -> pyam.IamDataFrame else: data = extract_results(results, technologies) - else: + elif isinstance(result["osemosys_param"], list): results = {} for p in result["osemosys_param"]: - path_name = os.path.join(results_path, p + ".csv") - results[p] = read_file(path_name) + results[p] = read_file(results_path, p, config["region"]) + if "trade_tech" in result.keys(): technologies = result["trade_tech"] data = calculate_trade(results, technologies) + else: + name = result["iamc_variable"] + raise ValueError(f"No data found for {name}") + + else: + name = result["iamc_variable"] + msg = f"Error in configuration file for entry {name}. The `osemosys_param` key must be a string or a list" + raise ValueError(msg) + if "transform" in result.keys(): if result["transform"] == "abs": data["VALUE"] = data["VALUE"].abs() From c6bcb5a32b251473f50069b999ea38a7f31a01e0 Mon Sep 17 00:00:00 2001 From: Will Usher Date: Tue, 11 Jul 2023 15:39:00 +0200 Subject: [PATCH 2/5] Return countries names in lower case --- tests/test_main.py | 40 ++++----- tests/test_resultify.py | 184 ++++++++++++++++++++-------------------- 2 files changed, 112 insertions(+), 112 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index b5735ef..e3466b8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -19,10 +19,10 @@ def test_main_input(): data = pd.DataFrame( [ - ["AUSTRIA", "Price|Primary Energy|Biomass", 2015, 3.15], - ["AUSTRIA", "Price|Primary Energy|Biomass", 2016, 4.2], - ["BELGIUM", "Price|Primary Energy|Biomass", 2015, 1.785], - ["BELGIUM", "Price|Primary Energy|Biomass", 2016, 1.890], + ["Austria", "Price|Primary Energy|Biomass", 2015, 3.15], + ["Austria", "Price|Primary Energy|Biomass", 2016, 4.2], + ["Belgium", "Price|Primary Energy|Biomass", 2015, 1.785], + ["Belgium", "Price|Primary Energy|Biomass", 2016, 1.890], ], columns=["region", "variable", "year", "value"], ) @@ -50,18 +50,18 @@ def test_main_result(): data = pd.DataFrame( [ - ["AUSTRIA", "Capacity|Electricity", 2015, 0.446776], - ["BELGIUM", "Capacity|Electricity", 2016, 0.184866], - ["BULGARIA", "Capacity|Electricity", 2015, 4.141], - ["CYPRUS", "Capacity|Electricity", 2015, 0.3904880555817921], - ["CZECHIA", "Capacity|Electricity", 2015, 0.299709], - ["DENMARK", "Capacity|Electricity", 2015, 0.0005], - ["ESTONIA", "Capacity|Electricity", 2015, 0.006], - ["FINLAND", "Capacity|Electricity", 2015, 0.0263], - ["FRANCE", "Capacity|Electricity", 2015, 0.47835], - ["GERMANY", "Capacity|Electricity", 2015, 9.62143], - ["SPAIN", "Capacity|Electricity", 2015, 7.7308], - ["SWITZERLAND", "Capacity|Electricity", 2026, 0.004563975391582646], + ["Austria", "Capacity|Electricity", 2015, 0.446776], + ["Belgium", "Capacity|Electricity", 2016, 0.184866], + ["Bulgaria", "Capacity|Electricity", 2015, 4.141], + ["Cyprus", "Capacity|Electricity", 2015, 0.3904880555817921], + ["Czechia", "Capacity|Electricity", 2015, 0.299709], + ["Denmark", "Capacity|Electricity", 2015, 0.0005], + ["Estonia", "Capacity|Electricity", 2015, 0.006], + ["Finland", "Capacity|Electricity", 2015, 0.0263], + ["France", "Capacity|Electricity", 2015, 0.47835], + ["Germany", "Capacity|Electricity", 2015, 9.62143], + ["Spain", "Capacity|Electricity", 2015, 7.7308], + ["Switzerland", "Capacity|Electricity", 2026, 0.004563975391582646], ], columns=["region", "variable", "year", "value"], ) @@ -92,10 +92,10 @@ def test_main_result_capture(): data = pd.DataFrame( [ - ["AUSTRIA", "Carbon Capture|Biomass", 2026, 7.573069442598169], - ["AUSTRIA", "Carbon Capture|Biomass", 2027, 7.766777427515737], - ["BELGIUM", "Carbon Capture|Biomass", 2026, 2.24498280006968], - ["BELGIUM", "Carbon Capture|Biomass", 2027, 6.746886436926597], + ["Austria", "Carbon Capture|Biomass", 2026, 7.573069442598169], + ["Austria", "Carbon Capture|Biomass", 2027, 7.766777427515737], + ["Belgium", "Carbon Capture|Biomass", 2026, 2.24498280006968], + ["Belgium", "Carbon Capture|Biomass", 2027, 6.746886436926597], ], columns=["region", "variable", "year", "value"], ) diff --git a/tests/test_resultify.py b/tests/test_resultify.py index 8a2e251..5beb80b 100644 --- a/tests/test_resultify.py +++ b/tests/test_resultify.py @@ -61,16 +61,16 @@ def test_filter_emission(self): actual = filter_emission_tech(input_data, emission) data = [ - ["AUSTRIA", 2026, -6244.862561], - ["AUSTRIA", 2027, -6529.532083], - ["AUSTRIA", 2030, 3043.148835], - ["AUSTRIA", 2031, 2189.064681], - ["AUSTRIA", 2032, 2315.821267], - ["BELGIUM", 2026, -2244.982800], - ["BELGIUM", 2027, -6746.886437], - ["BULGARIA", 2030, 11096.556931], - ["BULGARIA", 2031, 11069.257141], - ["BULGARIA", 2032, 11041.957354], + ["Austria", 2026, -6244.862561], + ["Austria", 2027, -6529.532083], + ["Austria", 2030, 3043.148835], + ["Austria", 2031, 2189.064681], + ["Austria", 2032, 2315.821267], + ["Belgium", 2026, -2244.982800], + ["Belgium", 2027, -6746.886437], + ["Bulgaria", 2030, 11096.556931], + ["Bulgaria", 2031, 11069.257141], + ["Bulgaria", 2032, 11041.957354], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -87,10 +87,10 @@ def test_filter_tech_emission(self): actual = filter_emission_tech(input_data, emission, tech) data = [ - ["AUSTRIA", 2026, -7573.069442598169], - ["AUSTRIA", 2027, -7766.777427515737], - ["BELGIUM", 2026, -2244.98280006968], - ["BELGIUM", 2027, -6746.886436926597], + ["Austria", 2026, -7573.069442598169], + ["Austria", 2027, -7766.777427515737], + ["Belgium", 2026, -2244.98280006968], + ["Belgium", 2027, -6746.886436926597], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -137,24 +137,24 @@ def test_filter_capacity(self): actual = filter_capacity(input_data, technologies) data = [ - ["AUSTRIA", 2015, 26.324108350683794], - ["AUSTRIA", 2016, 26.324108350683794], - ["AUSTRIA", 2017, 26.324108350683794], - ["AUSTRIA", 2018, 26.324108350683787], - ["AUSTRIA", 2019, 26.324108350683794], - ["BELGIUM", 2016, 141.0], - ["BULGARIA", 2015, 1.423512], - ["CZECHIA", 2015, 329.5950809], - ["DENMARK", 2015, 0.0031536], - ["ESTONIA", 2015, 28.512108], - ["FINLAND", 2015, 0.296581102], - ["FRANCE", 2015, 72.25974846], - ["SPAIN", 2015, 26.75595496], - ["SWITZERLAND", 2047, 69.9750212433476], - ["SWITZERLAND", 2048, 91.45662886581975], - ["SWITZERLAND", 2049, 76.86770297185006], - ["SWITZERLAND", 2050, 70.86078033897608], - ["SWITZERLAND", 2051, 53.88447040760964], + ["Austria", 2015, 26.324108350683794], + ["Austria", 2016, 26.324108350683794], + ["Austria", 2017, 26.324108350683794], + ["Austria", 2018, 26.324108350683787], + ["Austria", 2019, 26.324108350683794], + ["Belgium", 2016, 141.0], + ["Bulgaria", 2015, 1.423512], + ["Czechia", 2015, 329.5950809], + ["Denmark", 2015, 0.0031536], + ["Estonia", 2015, 28.512108], + ["Finland", 2015, 0.296581102], + ["France", 2015, 72.25974846], + ["Spain", 2015, 26.75595496], + ["Switzerland", 2047, 69.9750212433476], + ["Switzerland", 2048, 91.45662886581975], + ["Switzerland", 2049, 76.86770297185006], + ["Switzerland", 2050, 70.86078033897608], + ["Switzerland", 2051, 53.88447040760964], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -169,11 +169,11 @@ def test_filter_primary_bm(self): actual = filter_capacity(input_data, technologies) data = [ - ["AUSTRIA", 2015, 26.324108350683794], - ["AUSTRIA", 2016, 26.324108350683794], - ["AUSTRIA", 2017, 26.324108350683794], - ["AUSTRIA", 2018, 26.324108350683787], - ["AUSTRIA", 2019, 26.324108350683794], + ["Austria", 2015, 26.324108350683794], + ["Austria", 2016, 26.324108350683794], + ["Austria", 2017, 26.324108350683794], + ["Austria", 2018, 26.324108350683787], + ["Austria", 2019, 26.324108350683794], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -188,11 +188,11 @@ def test_filter_primary_co(self): actual = filter_capacity(input_data, technologies) data = [ - ["SWITZERLAND", 2047, 69.9750212433476], - ["SWITZERLAND", 2048, 91.45662886581975], - ["SWITZERLAND", 2049, 76.86770297185006], - ["SWITZERLAND", 2050, 70.86078033897608], - ["SWITZERLAND", 2051, 53.88447040760964], + ["Switzerland", 2047, 69.9750212433476], + ["Switzerland", 2048, 91.45662886581975], + ["Switzerland", 2049, 76.86770297185006], + ["Switzerland", 2050, 70.86078033897608], + ["Switzerland", 2051, 53.88447040760964], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -207,7 +207,7 @@ def test_filter_primary_ng(self): actual = filter_capacity(input_data, technologies) data = [ - ["BELGIUM", 2016, 141.0], + ["Belgium", 2016, 141.0], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -222,7 +222,7 @@ def test_filter_primary_go(self): actual = filter_capacity(input_data, technologies) data = [ - ["BULGARIA", 2015, 1.423512], + ["Bulgaria", 2015, 1.423512], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -236,7 +236,7 @@ def test_filter_primary_hy(self): actual = filter_capacity(input_data, technologies) data = [ - ["CZECHIA", 2015, 3.3637616987287244], + ["Czechia", 2015, 3.3637616987287244], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -250,7 +250,7 @@ def test_filter_primary_nu(self): actual = filter_capacity(input_data, technologies) data = [ - ["CZECHIA", 2015, 326.2313192401038], + ["Czechia", 2015, 326.2313192401038], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -264,7 +264,7 @@ def test_filter_primary_oc(self): actual = filter_capacity(input_data, technologies) data = [ - ["DENMARK", 2015, 0.0031536000000000003], + ["Denmark", 2015, 0.0031536000000000003], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -279,7 +279,7 @@ def test_filter_primary_oi(self): actual = filter_capacity(input_data, technologies) data = [ - ["ESTONIA", 2015, 28.512107999999998], + ["Estonia", 2015, 28.512107999999998], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -294,7 +294,7 @@ def test_filter_primary_so(self): actual = filter_capacity(input_data, technologies) data = [ - ["SPAIN", 2015, 26.75595496070811], + ["Spain", 2015, 26.75595496070811], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -309,8 +309,8 @@ def test_filter_primary_wi(self): actual = filter_capacity(input_data, technologies) data = [ - ["FINLAND", 2015, 0.29658110158442175], - ["FRANCE", 2015, 72.25974845531343], + ["Finland", 2015, 0.29658110158442175], + ["France", 2015, 72.25974845531343], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -329,11 +329,11 @@ def test_filter_secondary_bm(self): actual = filter_capacity(input_data, technologies) data = [ - ["AUSTRIA", 2042, 0.6636346353894057], - ["AUSTRIA", 2043, 1.3300518531620575], - ["AUSTRIA", 2044, 1.9992691432067637], - ["AUSTRIA", 2045, 2.6713041901899794], - ["AUSTRIA", 2046, 3.4778527409137996], + ["Austria", 2042, 0.6636346353894057], + ["Austria", 2043, 1.3300518531620575], + ["Austria", 2044, 1.9992691432067637], + ["Austria", 2045, 2.6713041901899794], + ["Austria", 2046, 3.4778527409137996], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -348,8 +348,8 @@ def test_filter_final_energy(self): actual = filter_final_energy(input_data, fuels) data = [ - ["AUSTRIA", 2015, 227.5944502], - ["BELGIUM", 2016, 296.0570016], + ["Austria", 2015, 227.5944502], + ["Belgium", 2016, 296.0570016], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -368,18 +368,18 @@ def test_filter_inst_capacity(self): actual = filter_capacity(input_data, technologies) data = [ - ["AUSTRIA", 2015, 0.446776], - ["BELGIUM", 2016, 0.184866], - ["BULGARIA", 2015, 4.141], - ["CYPRUS", 2015, 0.3904880555817921], - ["CZECHIA", 2015, 0.299709], - ["DENMARK", 2015, 0.0005], - ["ESTONIA", 2015, 0.006], - ["FINLAND", 2015, 0.0263], - ["FRANCE", 2015, 0.47835], - ["GERMANY", 2015, 9.62143], - ["SPAIN", 2015, 7.7308], - ["SWITZERLAND", 2026, 0.004563975391582646], + ["Austria", 2015, 0.446776], + ["Belgium", 2016, 0.184866], + ["Bulgaria", 2015, 4.141], + ["Cyprus", 2015, 0.3904880555817921], + ["Czechia", 2015, 0.299709], + ["Denmark", 2015, 0.0005], + ["Estonia", 2015, 0.006], + ["Finland", 2015, 0.0263], + ["France", 2015, 0.47835], + ["Germany", 2015, 9.62143], + ["Spain", 2015, 7.7308], + ["Switzerland", 2026, 0.004563975391582646], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -401,9 +401,9 @@ def test_filter_inst_capacity_bio(self): actual = filter_capacity(input_data, technologies) data = [ - ["AUSTRIA", 2015, 0.446776], - ["BELGIUM", 2016, 0.184866], - ["FRANCE", 2015, 0.47835], + ["Austria", 2015, 0.446776], + ["Belgium", 2016, 0.184866], + ["France", 2015, 0.47835], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -417,7 +417,7 @@ def test_filter_inst_capacity_coal(self): actual = filter_capacity(input_data, technologies) data = [ - ["BULGARIA", 2015, 4.141], + ["Bulgaria", 2015, 4.141], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -431,7 +431,7 @@ def test_filter_inst_capacity_gas(self): actual = filter_capacity(input_data, technologies) data = [ - ["GERMANY", 2015, 9.62143], + ["Germany", 2015, 9.62143], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -445,7 +445,7 @@ def test_filter_inst_capacity_geo(self): actual = filter_capacity(input_data, technologies) data = [ - ["SWITZERLAND", 2026, 0.004563975391582646], + ["Switzerland", 2026, 0.004563975391582646], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -459,7 +459,7 @@ def test_filter_inst_capacity_hydro(self): actual = filter_capacity(input_data, technologies) data = [ - ["CZECHIA", 2015, 0.299709], + ["Czechia", 2015, 0.299709], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -473,7 +473,7 @@ def test_filter_inst_capacity_nuclear(self): actual = filter_capacity(input_data, technologies) data = [ - ["SPAIN", 2015, 7.7308], + ["Spain", 2015, 7.7308], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -487,7 +487,7 @@ def test_filter_inst_capacity_ocean(self): actual = filter_capacity(input_data, technologies) data = [ - ["DENMARK", 2015, 0.0005], + ["Denmark", 2015, 0.0005], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -501,7 +501,7 @@ def test_filter_inst_capacity_oil(self): actual = filter_capacity(input_data, technologies) data = [ - ["CYPRUS", 2015, 0.3904880555817921], + ["Cyprus", 2015, 0.3904880555817921], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -515,7 +515,7 @@ def test_filter_inst_capacity_solar(self): actual = filter_capacity(input_data, technologies) data = [ - ["ESTONIA", 2015, 0.006], + ["Estonia", 2015, 0.006], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -529,7 +529,7 @@ def test_filter_inst_capacity_wi_offshore(self): actual = filter_capacity(input_data, technologies) data = [ - ["FINLAND", 2015, 0.0263], + ["Finland", 2015, 0.0263], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -545,10 +545,10 @@ def test_price_bm(self): actual = filter_capacity(input_data, commodity) data = [ - ["AUSTRIA", 2015, 3.0], - ["AUSTRIA", 2016, 4.0], - ["BELGIUM", 2015, 1.7], - ["BELGIUM", 2016, 1.8], + ["Austria", 2015, 3.0], + ["Austria", 2016, 4.0], + ["Belgium", 2015, 1.7], + ["Belgium", 2016, 1.8], ] expected = pd.DataFrame(data=data, columns=["REGION", "YEAR", "VALUE"]) @@ -565,7 +565,7 @@ def test_iso_to_country_iso2start(self): actual = iso_to_country("iso2_start", techs, "TotalCapacityAnnual") - expected = ["NIGERIA", "GERMANY", "NIGERIA", ""] + expected = ["Nigeria", "Germany", "Nigeria", ""] assert actual == expected @@ -574,7 +574,7 @@ def test_iso_to_country_iso3start(self): actual = iso_to_country("iso3_start", techs, "TotalCapacityAnnual") - expected = ["NIGERIA", "BAHAMAS", "TURKS AND CAICOS ISLANDS", ""] + expected = ["Nigeria", "Bahamas", "Turks and Caicos Islands", ""] assert actual == expected @@ -583,7 +583,7 @@ def test_iso_to_country_iso2end(self): actual = iso_to_country("iso2_end", techs, "TotalCapacityAnnual") - expected = ["NIGERIA", "GERMANY", "NIGERIA", ""] + expected = ["Nigeria", "Germany", "Nigeria", ""] assert actual == expected @@ -592,7 +592,7 @@ def test_iso_to_country_iso3end(self): actual = iso_to_country("iso3_end", techs, "TotalCapacityAnnual") - expected = ["NIGERIA", "BAHAMAS", "TURKS AND CAICOS ISLANDS", ""] + expected = ["Nigeria", "Bahamas", "Turks and Caicos Islands", ""] assert actual == expected @@ -601,7 +601,7 @@ def test_iso_to_country_iso2middle(self): actual = iso_to_country("iso2_5", techs, "TotalCapacityAnnual") - expected = ["NIGERIA", "GERMANY", "", ""] + expected = ["Nigeria", "Germany", "", ""] assert actual == expected @@ -610,6 +610,6 @@ def test_iso_to_country_iso3middle(self): actual = iso_to_country("iso3_7", techs, "TotalCapacityAnnual") - expected = ["GHANA", "SAINT LUCIA", "", ""] + expected = ["Ghana", "Saint Lucia", "", ""] assert actual == expected From 9c80d4304f01aa13d98e2925f840f5d6567b731b Mon Sep 17 00:00:00 2001 From: Will Usher Date: Wed, 12 Jul 2023 10:12:06 +0200 Subject: [PATCH 3/5] Add acknowledgement to IAM COMPACT --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a51a21..8758160 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ # osemosys2iamc +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7473185.svg)](https://doi.org/10.5281/zenodo.7473185) + Convert OSeMOSYS results to IAMC format ## Acknowledgements -This work was financially supported by the European Union’s Horizon 2020 research and innovation programme under the grant agreement No 101022622 ([European Climate and Energy Modelling Forum ECEMF](https://doi.org/10.3030/101022622)). +This work was financially supported by: + +- The European Union’s Horizon 2020 research and innovation programme under the grant agreement No 101022622 ([European Climate and Energy Modelling Forum ECEMF](https://doi.org/10.3030/101022622)) +- The [IAM COMPACT](https://doi.org/10.3030/101056306) project has received funding from the European Union’s HORIZON EUROPE Research and Innovation Programme under grant agreement No 101056306 ## Install from Github repository From 34d6b5bd61c6199378c0dd4a58117d23a73b86c1 Mon Sep 17 00:00:00 2001 From: Will Usher Date: Wed, 12 Jul 2023 10:22:55 +0200 Subject: [PATCH 4/5] Add test for trade and override for UK and EL iso2 codes --- src/osemosys2iamc/resultify.py | 23 +++++++++----- tests/fixtures/trade/config_trade.yaml | 11 +++++++ tests/test_main.py | 43 ++++++++++++++++++++++++++ tests/test_resultify.py | 6 ++++ 4 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/trade/config_trade.yaml diff --git a/src/osemosys2iamc/resultify.py b/src/osemosys2iamc/resultify.py index 718cd8b..b9da574 100644 --- a/src/osemosys2iamc/resultify.py +++ b/src/osemosys2iamc/resultify.py @@ -26,6 +26,11 @@ import matplotlib.dates as mdates import re +# Creates an alias for United Kingdom and Greece using alternate 2-letter code +# (see issue https://github.com/OSeMOSYS/osemosys2iamc/issues/33) +countries_by_alpha2["UK"] = countries_by_alpha2["GB"] +countries_by_alpha2["EL"] = countries_by_alpha2["GR"] + def iso_to_country( iso_format: str, index: List[str], osemosys_param: str @@ -472,6 +477,7 @@ def main(config: Dict, inputs_path: str, results_path: str) -> pyam.IamDataFrame elif isinstance(result["osemosys_param"], list): results = {} + unit = result["unit"] for p in result["osemosys_param"]: results[p] = read_file(results_path, p, config["region"]) @@ -510,15 +516,18 @@ def main(config: Dict, inputs_path: str, results_path: str) -> pyam.IamDataFrame except KeyError: pass - all_data = pyam.concat(blob) + if len(blob) > 0: + all_data = pyam.concat(blob) - all_data = all_data.convert_unit("PJ/yr", to="EJ/yr") - all_data = all_data.convert_unit("ktCO2/yr", to="Mt CO2/yr", factor=0.001) - all_data = all_data.convert_unit("MEUR_2015/PJ", to="EUR_2020/GJ", factor=1.05) - all_data = all_data.convert_unit("kt CO2/yr", to="Mt CO2/yr") + all_data = all_data.convert_unit("PJ/yr", to="EJ/yr") + all_data = all_data.convert_unit("ktCO2/yr", to="Mt CO2/yr", factor=0.001) + all_data = all_data.convert_unit("MEUR_2015/PJ", to="EUR_2020/GJ", factor=1.05) + all_data = all_data.convert_unit("kt CO2/yr", to="Mt CO2/yr") - all_data = pyam.IamDataFrame(all_data) - return all_data + all_data = pyam.IamDataFrame(all_data) + return all_data + else: + raise ValueError("No data found") def aggregate(func): diff --git a/tests/fixtures/trade/config_trade.yaml b/tests/fixtures/trade/config_trade.yaml new file mode 100644 index 0000000..79d7d41 --- /dev/null +++ b/tests/fixtures/trade/config_trade.yaml @@ -0,0 +1,11 @@ +model: OSeMBE v1.0.0 +scenario: DIAG-C400-lin-ResidualFossil +region: 'iso2_start' #iso2_x, iso3_x, from_csv, or a name of a country/region [substitute x with start, end, or a positive number] +results: +- iamc_variable: Trade|Secondary Energy|Electricity|Volume + osemosys_param: + - UseByTechnology + - ProductionByTechnologyAnnual + trade_tech: + - (?=^.{2}(EL))^((?!00).)*$ + unit: PJ/yr diff --git a/tests/test_main.py b/tests/test_main.py index e3466b8..8410a3b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -108,3 +108,46 @@ def test_main_result_capture(): ) assert_iamframe_equal(actual, expected) + + +def test_main_trade(): + """Test operation of trade filter + + Config + ------ + - iamc_variable: Trade|Secondary Energy|Electricity|Volume + osemosys_param: + - UseByTechnology + - ProductionByTechnologyAnnual + trade_tech: + - (?=^.{2}(EL))^((?!00).)*$ + unit: PJ/yr + + """ + + config_path = os.path.join("tests", "fixtures", "trade", "config_trade.yaml") + inputs_path = os.path.join("tests", "fixtures", "trade") + results_path = os.path.join("tests", "fixtures", "trade") + + with open(config_path, "r") as config_file: + config = load(config_file, Loader=SafeLoader) + + actual = main(config, inputs_path, results_path) + + data = pd.DataFrame( + [ + ["Austria", "Trade|Secondary Energy|Electricity|Volume", 2010, -0.024824], + ["Austria", "Trade|Secondary Energy|Electricity|Volume", 2011, -0.024924], + ["Austria", "Trade|Secondary Energy|Electricity|Volume", 2012, -0.025024], + ], + columns=["region", "variable", "year", "value"], + ) + + expected = IamDataFrame( + data, + model="OSeMBE v1.0.0", + scenario="DIAG-C400-lin-ResidualFossil", + unit="EJ/yr", + ) + + assert_iamframe_equal(actual, expected) diff --git a/tests/test_resultify.py b/tests/test_resultify.py index 5beb80b..264e970 100644 --- a/tests/test_resultify.py +++ b/tests/test_resultify.py @@ -613,3 +613,9 @@ def test_iso_to_country_iso3middle(self): expected = ["Ghana", "Saint Lucia", "", ""] assert actual == expected + + def test_iso_to_country_uk(self): + techs = ["UKNGA"] + actual = iso_to_country("iso2_start", techs, "TotalCapacityAnnual") + expected = ["United Kingdom of Great Britain and Northern Ireland"] + assert actual == expected From b5b81d4039f285fd4c2d56584f4b645e3890e5e7 Mon Sep 17 00:00:00 2001 From: Will Usher Date: Wed, 12 Jul 2023 10:27:55 +0200 Subject: [PATCH 5/5] Add fixture files for trade test --- tests/fixtures/trade/ProductionByTechnologyAnnual.csv | 4 ++++ tests/fixtures/trade/UseByTechnology.csv | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 tests/fixtures/trade/ProductionByTechnologyAnnual.csv create mode 100644 tests/fixtures/trade/UseByTechnology.csv diff --git a/tests/fixtures/trade/ProductionByTechnologyAnnual.csv b/tests/fixtures/trade/ProductionByTechnologyAnnual.csv new file mode 100644 index 0000000..2c7a029 --- /dev/null +++ b/tests/fixtures/trade/ProductionByTechnologyAnnual.csv @@ -0,0 +1,4 @@ +REGION,TECHNOLOGY,FUEL,YEAR,VALUE +REGION1,ATEL1234,ATEL,2010,26.324108350683794 +REGION1,ATEL1234,ATEL,2011,26.324108350683794 +REGION1,ATEL1234,ATEL,2012,26.324108350683794 diff --git a/tests/fixtures/trade/UseByTechnology.csv b/tests/fixtures/trade/UseByTechnology.csv new file mode 100644 index 0000000..8735300 --- /dev/null +++ b/tests/fixtures/trade/UseByTechnology.csv @@ -0,0 +1,4 @@ +REGION,TIMESLICE,TECHNOLOGY,FUEL,YEAR,VALUE +REGION1,ID,ATEL1234,ATEL,2010,1.5 +REGION1,ID,ATEL1234,ATEL,2011,1.4 +REGION1,ID,ATEL1234,ATEL,2012,1.3