Skip to content

Commit

Permalink
updated ReadCplex tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@trevorb1
trevorb1 committed Aug 22, 2023
1 parent 81fbaba commit c413ae6
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 197 deletions.
12 changes: 7 additions & 5 deletions src/otoole/results/results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,8 @@ def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
Arguments
---------
file_path : str
user_config : Dict[str, Dict]
file_path : Union[str, TextIO]
"""
df = pd.read_xml(file_path, xpath=".//variable", parser="etree")
df[["Variable", "Index"]] = df["name"].str.split("(", expand=True)
Expand All @@ -206,7 +207,8 @@ def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
Arguments
---------
file_path : str
user_config : Dict[str, Dict]
file_path : Union[str, TextIO]
"""
df = pd.read_csv(
file_path,
Expand All @@ -227,8 +229,8 @@ class ReadCbc(ReadWideResults):
Arguments
---------
user_config
results_config
user_config : Dict[str, Dict]
results_config : Dict[str, Dict]
"""

def _convert_to_dataframe(self, file_path: Union[str, TextIO]) -> pd.DataFrame:
Expand Down Expand Up @@ -267,7 +269,7 @@ class ReadGlpk(ReadWideResults):
Arguments
---------
user_config
user_config : Dict[str, Dict]
glpk_model: Union[str, TextIO]
Path to GLPK model file. Can be created using the `--wglp` flag.
"""
Expand Down
301 changes: 109 additions & 192 deletions tests/test_read_strategies.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from pytest import mark, raises

from otoole.exceptions import OtooleDeprecationError, OtooleError
from otoole.preprocess.longify_data import check_datatypes

# from otoole.preprocess.longify_data import check_datatypes
from otoole.read_strategies import ReadCsv, ReadDatafile, ReadExcel, ReadMemory
from otoole.results.results import (
ReadCbc,
Expand All @@ -23,212 +24,128 @@

class TestReadCplex:

cplex_empty = (
"AnnualFixedOperatingCost REGION AOBACKSTOP 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"
)
cplex_short = "AnnualFixedOperatingCost REGION CDBACKSTOP 0.0 0.0 137958.8400384134 305945.38410619126 626159.9611543404 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0"
cplex_long = "RateOfActivity REGION S1D1 CGLFRCFURX 1 0.0 0.0 0.0 0.0 0.0 0.3284446367303371 0.3451714779880536 0.3366163200621617 0.3394945166233896 0.3137488154250392 0.28605725055560716 0.2572505015401749 0.06757558148965725 0.0558936625751148 0.04330608461292407 0.0"

cplex_mid_empty = (
pd.DataFrame(
data=[],
columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
)
.astype({"VALUE": float})
.set_index(["REGION", "TECHNOLOGY", "YEAR"])
)

cplex_mid_short = pd.DataFrame(
data=[
["REGION", "CDBACKSTOP", 2017, 137958.8400384134],
["REGION", "CDBACKSTOP", 2018, 305945.38410619126],
["REGION", "CDBACKSTOP", 2019, 626159.9611543404],
],
columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
).set_index(["REGION", "TECHNOLOGY", "YEAR"])

cplex_mid_long = pd.DataFrame(
data=[
["REGION", "S1D1", "CGLFRCFURX", 1, 2020, 0.3284446367303371],
["REGION", "S1D1", "CGLFRCFURX", 1, 2021, 0.3451714779880536],
["REGION", "S1D1", "CGLFRCFURX", 1, 2022, 0.3366163200621617],
["REGION", "S1D1", "CGLFRCFURX", 1, 2023, 0.3394945166233896],
["REGION", "S1D1", "CGLFRCFURX", 1, 2024, 0.3137488154250392],
["REGION", "S1D1", "CGLFRCFURX", 1, 2025, 0.28605725055560716],
["REGION", "S1D1", "CGLFRCFURX", 1, 2026, 0.2572505015401749],
["REGION", "S1D1", "CGLFRCFURX", 1, 2027, 0.06757558148965725],
["REGION", "S1D1", "CGLFRCFURX", 1, 2028, 0.0558936625751148],
["REGION", "S1D1", "CGLFRCFURX", 1, 2029, 0.04330608461292407],
],
columns=[
"REGION",
"TIMESLICE",
"TECHNOLOGY",
"MODE_OF_OPERATION",
"YEAR",
"VALUE",
],
).set_index(["REGION", "TIMESLICE", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR"])
cplex_data = """<?xml version = "1.0" encoding="UTF-8" standalone="yes"?>
<CPLEXSolution version="1.2">
<header
problemName="model.lp"
objectiveValue="3942.194792652062"
solutionTypeValue="1"
solutionTypeString="basic"
solutionStatusValue="1"
solutionStatusString="optimal"
solutionMethodString="dual"
primalFeasible="1"
dualFeasible="1"
simplexIterations="3011"
writeLevel="1"/>
<quality
epRHS="9.9999999999999995e-07"
epOpt="9.9999999999999995e-07"
maxPrimalInfeas="1.4210854715202004e-13"
maxDualInfeas="1.9618827037793883e-12"
maxPrimalResidual="5.3193968022437806e-12"
maxDualResidual="5.6354920729972946e-13"
maxX="1331.0025494901022"
maxPi="525.09145670749103"
maxSlack="9998.6832048983561"
maxRedCost="999999.00000000105"
kappa="57958.643677766071"/>
<linearConstraints>
<constraint name="CAa4_Constraint_Capacity(SIMPLICITY,ID,BACKSTOP1,2014)" index="0" status="BS" slack="-0" dual="0"/>
<constraint name="CAa4_Constraint_Capacity(SIMPLICITY,ID,BACKSTOP1,2015)" index="1" status="BS" slack="-0" dual="0"/>
<constraint name="CAa4_Constraint_Capacity(SIMPLICITY,ID,BACKSTOP1,2016)" index="2" status="BS" slack="-0" dual="0"/>
<constraint name="EBa11_EnergyBalanceEachTS5(SIMPLICITY,IN,DSL,2014)" index="5400" status="LL" slack="0" dual="-2.4281963661332018e-14"/>
<constraint name="EBa11_EnergyBalanceEachTS5(SIMPLICITY,IN,DSL,2015)" index="5401" status="LL" slack="0" dual="-4.4565525073113654e-15"/>
<constraint name="EBa11_EnergyBalanceEachTS5(SIMPLICITY,IN,DSL,2016)" index="5402" status="LL" slack="0" dual="7.5386542413397867e-15"/>
<constraint name="TCC1_TotalAnnualMaxCapacityConstraint(SIMPLICITY,HYD2,2014)" index="11529" status="BS" slack="0.10000000000000001" dual="0"/>
<constraint name="TCC1_TotalAnnualMaxCapacityConstraint(SIMPLICITY,HYD2,2015)" index="11530" status="BS" slack="0.10000000000000001" dual="0"/>
<constraint name="TCC1_TotalAnnualMaxCapacityConstraint(SIMPLICITY,HYD2,2016)" index="11531" status="BS" slack="0.10000000000000001" dual="0"/>
</linearConstraints>
<variables>
<variable name="NewCapacity(SIMPLICITY,ETHPLANT,2014)" index="54" status="LL" value="0" reducedCost="9.8033882532023107"/>
<variable name="NewCapacity(SIMPLICITY,ETHPLANT,2015)" index="55" status="BS" value="0.030000000000000027" reducedCost="0"/>
<variable name="NewCapacity(SIMPLICITY,ETHPLANT,2016)" index="56" status="BS" value="0.030999999999999917" reducedCost="0"/>
<variable name="RateOfActivity(SIMPLICITY,ID,GRID_EXP,1,2014)" index="6048" status="LL" value="0" reducedCost="2.5520010388720782"/>
<variable name="RateOfActivity(SIMPLICITY,ID,GRID_EXP,2,2014)" index="6049" status="LL" value="0" reducedCost="2.5520010388720782"/>
<variable name="RateOfActivity(SIMPLICITY,ID,HYD1,1,2020)" index="6108" status="BS" value="0.25228800000000001" reducedCost="0"/>
<variable name="RateOfActivity(SIMPLICITY,ID,HYD1,1,2021)" index="6109" status="BS" value="0.25228800000000001" reducedCost="0"/>
<variable name="RateOfActivity(SIMPLICITY,ID,HYD1,1,2022)" index="6110" status="BS" value="0.25228800000000001" reducedCost="0"/>
</variables>
<objectiveValues>
<objective index="0" name="cost" value="3942.1947926520666"/>
</objectiveValues>
</CPLEXSolution>"""

dataframe_short = {
"AnnualFixedOperatingCost": pd.DataFrame(
data=[
["REGION", "CDBACKSTOP", 2017, 137958.8400384134],
["REGION", "CDBACKSTOP", 2018, 305945.3841061913],
["REGION", "CDBACKSTOP", 2019, 626159.9611543404],
],
columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
).set_index(["REGION", "TECHNOLOGY", "YEAR"])
}

dataframe_long = {
"RateOfActivity": pd.DataFrame(
data=[
["REGION", "S1D1", "CGLFRCFURX", 1, 2020, 0.3284446367303371],
["REGION", "S1D1", "CGLFRCFURX", 1, 2021, 0.3451714779880536],
["REGION", "S1D1", "CGLFRCFURX", 1, 2022, 0.3366163200621617],
["REGION", "S1D1", "CGLFRCFURX", 1, 2023, 0.3394945166233896],
["REGION", "S1D1", "CGLFRCFURX", 1, 2024, 0.3137488154250392],
["REGION", "S1D1", "CGLFRCFURX", 1, 2025, 0.28605725055560716],
["REGION", "S1D1", "CGLFRCFURX", 1, 2026, 0.2572505015401749],
["REGION", "S1D1", "CGLFRCFURX", 1, 2027, 0.06757558148965725],
["REGION", "S1D1", "CGLFRCFURX", 1, 2028, 0.0558936625751148],
["REGION", "S1D1", "CGLFRCFURX", 1, 2029, 0.04330608461292407],
],
columns=[
"REGION",
"TIMESLICE",
"TECHNOLOGY",
"MODE_OF_OPERATION",
"YEAR",
"VALUE",
def test_convert_to_dataframe(self, user_config):
input_file = self.cplex_data
reader = ReadCplex(user_config)
with StringIO(input_file) as file_buffer:
actual = reader._convert_to_dataframe(file_buffer)
# print(actual)
expected = pd.DataFrame(
[
["NewCapacity", "SIMPLICITY,ETHPLANT,2015", 0.030000000000000027],
["NewCapacity", "SIMPLICITY,ETHPLANT,2016", 0.030999999999999917],
["RateOfActivity", "SIMPLICITY,ID,HYD1,1,2020", 0.25228800000000001],
["RateOfActivity", "SIMPLICITY,ID,HYD1,1,2021", 0.25228800000000001],
["RateOfActivity", "SIMPLICITY,ID,HYD1,1,2022", 0.25228800000000001],
],
).set_index(["REGION", "TIMESLICE", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR"])
}

test_data = [
(cplex_short, dataframe_short),
(cplex_long, dataframe_long),
]

@mark.parametrize("cplex_input,expected", test_data, ids=["short", "long"])
def test_read_cplex_to_dataframe(self, cplex_input, expected, user_config):
cplex_reader = ReadCplex(user_config=user_config)

input_data = {
"YEAR": pd.DataFrame(data=list(range(2015, 2031, 1)), columns=["VALUE"]),
"REGION": pd.DataFrame(data=["REGION"], columns=["VALUE"]),
"TECHNOLOGY": pd.DataFrame(
data=["CDBACKSTOP", "CGLFRCFURX"], columns=["VALUE"]
),
"MODE_OF_OPERATION": pd.DataFrame(data=[1], columns=["VALUE"]),
"TIMESLICE": pd.DataFrame(data=["S1D1"], columns=["VALUE"]),
}

with StringIO(cplex_input) as file_buffer:
actual, _ = cplex_reader.read(file_buffer, input_data=input_data)
for name, item in actual.items():
pd.testing.assert_frame_equal(item, expected[name])

test_data_mid = [(cplex_short, cplex_mid_short), (cplex_long, cplex_mid_long)]

def test_read_empty_cplex_to_dataframe(self, user_config):
cplex_input = self.cplex_empty

cplex_reader = ReadCplex(user_config)
columns=["Variable", "Index", "Value"],
).astype({"Variable": str, "Index": str, "Value": float})

input_data = {
"YEAR": pd.DataFrame(data=list(range(2015, 2031, 1)), columns=["VALUE"])
}
pd.testing.assert_frame_equal(actual, expected)

with StringIO(cplex_input) as file_buffer:
data, _ = cplex_reader.read(file_buffer, input_data=input_data)
assert "AnnualFixedOperatingCost" in data
def test_solution_to_dataframe(self, user_config):
input_file = self.cplex_data
reader = ReadCplex(user_config)
with StringIO(input_file) as file_buffer:
actual = reader.read(file_buffer)
# print(actual)
expected = (
pd.DataFrame(
data=[],
[
["SIMPLICITY", "ETHPLANT", 2015, 0.030000000000000027],
["SIMPLICITY", "ETHPLANT", 2016, 0.030999999999999917],
],
columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
)
.astype({"REGION": str, "VALUE": float, "YEAR": int, "TECHNOLOGY": str})
.astype({"REGION": str, "TECHNOLOGY": str, "YEAR": int, "VALUE": float})
.set_index(["REGION", "TECHNOLOGY", "YEAR"])
)
actual = data["AnnualFixedOperatingCost"]
pd.testing.assert_frame_equal(actual, expected, check_index_type=False)

test_data_to_cplex = [
(cplex_empty, cplex_mid_empty),
(cplex_short, cplex_mid_short),
(cplex_long, cplex_mid_long),
]

@mark.parametrize(
"cplex_input,expected", test_data_to_cplex, ids=["empty", "short", "long"]
)
def test_convert_cplex_to_df(self, cplex_input, expected, user_config):

data = cplex_input.split("\t")
variable = data[0]
cplex_reader = ReadCplex(user_config=user_config)
actual = cplex_reader.convert_df([data], variable, 2015, 2030)
pd.testing.assert_frame_equal(actual, expected, check_index_type=False)

def test_convert_lines_to_df_empty(self, user_config):
pd.testing.assert_frame_equal(actual[0]["NewCapacity"], expected)

data = [
[
"AnnualFixedOperatingCost",
"REGION",
"AOBACKSTOP",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
"0",
]
]
variable = "AnnualFixedOperatingCost"
cplex_reader = ReadCplex(user_config)
actual = cplex_reader.convert_df(data, variable, 2015, 2023)
pd.testing.assert_frame_equal(
actual,
expected = (
pd.DataFrame(
data=[],
columns=["REGION", "TECHNOLOGY", "YEAR", "VALUE"],
[
["SIMPLICITY", "ID", "HYD1", 1, 2020, 0.25228800000000001],
["SIMPLICITY", "ID", "HYD1", 1, 2021, 0.25228800000000001],
["SIMPLICITY", "ID", "HYD1", 1, 2022, 0.25228800000000001],
],
columns=[
"REGION",
"TIMESLICE",
"TECHNOLOGY",
"MODE_OF_OPERATION",
"YEAR",
"VALUE",
],
)
.astype(
{
"REGION": str,
"TIMESLICE": str,
"TECHNOLOGY": str,
"MODE_OF_OPERATION": int,
"YEAR": int,
"VALUE": float,
}
)
.set_index(
["REGION", "TIMESLICE", "TECHNOLOGY", "MODE_OF_OPERATION", "YEAR"]
)
.astype({"REGION": str, "TECHNOLOGY": str, "YEAR": int, "VALUE": float})
.set_index(["REGION", "TECHNOLOGY", "YEAR"]),
check_index_type=False,
)

def test_check_datatypes_with_empty(self):

df = pd.DataFrame(data=[], columns=["REGION", "FUEL", "YEAR", "VALUE"])

parameter = "AccumulatedAnnualDemand"

config_dict = {
"AccumulatedAnnualDemand": {
"indices": ["REGION", "FUEL", "YEAR"],
"type": "param",
"dtype": float,
"default": 0,
},
"REGION": {"dtype": "str", "type": "set"},
"FUEL": {"dtype": "str", "type": "set"},
"YEAR": {"dtype": "int", "type": "set"},
}

actual = check_datatypes(df, config_dict, parameter)

expected = pd.DataFrame(
data=[], columns=["REGION", "FUEL", "YEAR", "VALUE"]
).astype({"REGION": str, "FUEL": str, "YEAR": int, "VALUE": float})

pd.testing.assert_frame_equal(actual, expected, check_index_type=False)
pd.testing.assert_frame_equal(actual[0]["RateOfActivity"], expected)


class TestReadGurobi:
Expand Down

0 comments on commit c413ae6

Please sign in to comment.