Skip to content

Commit

Permalink
building reopt_jl sysimage in install_reopt_jl
Browse files Browse the repository at this point in the history
lilycatolson committed Nov 10, 2023
1 parent 9ba626f commit 4ebb87f
Showing 6 changed files with 91 additions and 144 deletions.
20 changes: 9 additions & 11 deletions omf/models/microgridDesign2.py
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
from os.path import join as pJoin
import numpy as np
import pandas as pd
import xlwt
import plotly
import plotly.graph_objs as go
import omf
@@ -332,14 +333,11 @@ def work(modelDir, inputDict):
with open(pJoin(modelDir, "Scenario_test_POST.json"), "w") as jsonFile:
json.dump(scenario, jsonFile)

# Run REopt API script
###########todo: replace with reopt_jl call
# Run REopt API script *** => switched to REopt.jl
#REopt.run(pJoin(modelDir, 'Scenario_test_POST.json'), pJoin(modelDir, 'results.json'), inputDict['api_key'])
run_outages = True if outage_start_hour != 0 else False
reopt_jl.run_reopt_jl(modelDir, "Scenario_test_POST.json", convert=False, solver_in_filename=False,
outages=run_outages, max_runtime_s = 1000 )
with open(pJoin(modelDir, 'out_Scenario_test_POST.json')) as jsonFile:
#with open(pJoin(modelDir, 'results.json')) as jsonFile:
reopt_jl.run_reopt_jl(modelDir, "Scenario_test_POST.json", outages=run_outages, max_runtime_s = 1000 )
with open(pJoin(modelDir, 'results.json')) as jsonFile:
results = json.load(jsonFile)

#getting REoptInputs to access default input values more easily
@@ -348,9 +346,8 @@ def work(modelDir, inputDict):

#runID = results['outputs']['Scenario']['run_uuid']
#REopt.runResilience(runID, pJoin(modelDir, 'resultsResilience.json'), inputDict['api_key'])
#with open(pJoin(modelDir, 'resultsResilience.json')) as jsonFile:
if run_outages:
with open(pJoin(modelDir, 'outages_Scenario_test_POST.json')) as jsonFile:
with open(pJoin(modelDir, 'resultsResilience.json')) as jsonFile:
resultsResilience = json.load(jsonFile)

#potential nested function: define each outData variable as results key concatentation and add to outData
@@ -368,8 +365,9 @@ def addToOutData(key):
#resultsSubset = results['outputs']['Scenario']['Site']
outData['demandCostBAU' + indexString] = results['ElectricTariff']['lifecycle_demand_cost_after_tax_bau']#['total_demand_cost_bau_us_dollars']
if outData['demandCostBAU' + indexString] is None:
errMsg = results['messages'].get('error','API currently unavailable please try again later')
raise Exception('The REopt data analysis API by NREL had the following error: ' + errMsg)
#errMsg = results['messages'].get('error','API currently unavailable please try again later')
#raise Exception('The REopt data analysis API by NREL had the following error: ' + errMsg)
raise Exception('Error: results not received')

outData['demandCost' + indexString] = results['ElectricTariff']['lifecycle_demand_cost_after_tax']#['total_demand_cost_us_dollars']
outData['demandCostDiff' + indexString] = round(outData['demandCostBAU' + indexString] - outData['demandCost' + indexString],2)
@@ -643,7 +641,6 @@ def addToOutData(key):
# outData['']

#todo: decide on ProForma output type (excel, html, or both)
import xlwt #move to top

workbook = xlwt.Workbook()
worksheet = workbook.add_sheet("Results Summary and Inputs")
@@ -1098,6 +1095,7 @@ def makeGridLine(x,y,color,name):
outData["resilienceProbData" + indexString] = json.dumps(plotData, cls=plotly.utils.PlotlyJSONEncoder)
outData["resilienceProbLayout" + indexString] = json.dumps(plotlyLayout, cls=plotly.utils.PlotlyJSONEncoder)
if numCols == 1:
#todo: test reopt.jl updates with multiple loads
break # if we only have a single load, don't run an additional combined load shape run.

return outData
55 changes: 11 additions & 44 deletions omf/solvers/reopt_jl/README.md
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ julia> ]

__init __.py:
- run_reopt_jl(path, inputFile="", default=False, convert=True, outages=False, microgrid_only=False,
solver="HiGHS", solver_in_filename=True, max_runtime_s=None)
solver="HiGHS", max_runtime_s=None)

General paramters:
- path: directory containing inputFile ; output files written here as well
@@ -35,29 +35,25 @@ General paramters:
- max_runtime_s: default is None, otherwise times out after given number of seconds and returns local optimal value (may not be the global optimum)

Testing parameters:
- solver: set to HiGHS (best runtime performance) ; other available options: SCIP
- solver_in_filename: puts solver in filename if True
- solver: set to HiGHS (best runtime performance) ; other available options: SCIP
(add SCIP package to project [instructions above] & update imports in REoptSolver.jl in order to utilize)

Examples:
``` >>> run_reopt_jl(currentDir, inputFile="path/to/inputFile.json") ```
writes ouput file from REopt.jl to "currentDir/out_HiGHS_inputFile.json"
``` >>> run_reopt_jl(currentDir, inputFile="Scenario_test_POST.json") ```
writes ouput file from REopt.jl to "currentDir/results.json"

``` >>> run_reopt_jl(currentDir, default=True) ```
uses julia_default.json as input and writes ouput file from REopt.jl to "currentDir/out_julia_default.json"
uses julia_default.json as input and writes ouput file from REopt.jl to "currentDir/results.json"

``` >>> run_reopt_jl(currentDir, inputFile="path/to/inputFile.json", solver_in_filename=False) ```
writes output file from REopt.jl to "currentDir/out_inputFile.json"

``` >>> run_reopt_jl(currentDir, inputFile="path/to/inputFile.json", outages=True) ```
writes ouput file from REopt.jl to "currentDir/out_HiGHS_inputFile.json" and
writes outage output fie to "currentDir/outages_HiGHS_inputFile.json"
``` >>> run_reopt_jl(currentDir, inputFile="Scenario_test_POST.json", outages=True) ```
writes ouput file from REopt.jl to "currentDir/results.json" and
writes outage output fie to "currentDir/resultsResilience.json"

Testing usage (work in progress):

- __ init__.py:
- runAllSolvers(path, testName, fileName="", default=False, convert=True, outages=True,
solvers=["SCIP","HiGHS"], solver_in_filename=True, max_runtime_s=None,
get_cached=True )
solvers=["SCIP","HiGHS"], max_runtime_s=None, get_cached=True )

Usage: simlar to run_reopt_jl but takes in list of solvers and runs each one on the given test case ; outputs runtime comparisons

@@ -69,34 +65,5 @@ Inputs:
- convert : run_reopt_jl convert
- outages : run_reopt_jl outages
- solvers : list of solvers to call run_reopt_jl with
- solver_in_filename : run_reopt_jl solver_in_filename
- get_cached : loads previous output file from run_reopt_jl if found at path/out_fileName
- used to display previous results without re-running test

- test_outputs.py:
- html_comparison(all_tests)

Inputs:
- all_tests = [ (outPath, outagePath, testName, runtime, solver, outages, get_cached), ... ]
- outPath = path/out_fileName
- outagePath = path/outages_fileName
- testName = runAllSolvers testName
- runtime = runtime of test
- solver = current runAllSolvers solver (iterates through)
- outages = runAllSolvers outages
- get_cached = runAllSolvers get_cached

Outputs (can be altered based on testing goals - work in progress):
written to path/sample_comparison_test.html
- runtime comparison chart
- for each test case:
- general overview (parameters & runtime)
- microgrid overview
- financial performance overview
- load overview graph
- solar generation graph
- fossil generation graph
- battery charge source graph
- battery charge percentage graph
- resilience overview graph
- outage survival probability graph
- test_outputs.py => DEPRECATED (can be found in REopt_replacements in wiires repository)
25 changes: 24 additions & 1 deletion omf/solvers/reopt_jl/REoptSolver/Manifest.toml
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@

julia_version = "1.9.3"
manifest_format = "2.0"
project_hash = "0f3466f2c6876628d09be51d5a7ef29f3e074ef0"
project_hash = "551ae560fac0d92e3830e039edc6cd362abb050a"

[[deps.AbstractFFTs]]
deps = ["LinearAlgebra"]
@@ -344,6 +344,11 @@ git-tree-sha1 = "0e26e1737e94de57c858649dc28e482b6c87d341"
uuid = "0329782f-3d07-4b52-b9f6-d3137cf03c7a"
version = "1.0.1"

[[deps.Glob]]
git-tree-sha1 = "97285bbd5230dd766e9ef6749b80fc617126d496"
uuid = "c27321d9-0574-5035-807b-f59d2c89b15c"
version = "1.3.1"

[[deps.Graphics]]
deps = ["Colors", "LinearAlgebra", "NaNMath"]
git-tree-sha1 = "d61890399bc535850c4bf08e4e0d3a7ad0f21cbd"
@@ -727,6 +732,12 @@ git-tree-sha1 = "f3e45027ea0f44a2725fbedfdb7ed118d5deec8d"
uuid = "58948b4f-47e0-5654-a9ad-f609743f8632"
version = "901.300.0+0"

[[deps.PackageCompiler]]
deps = ["Artifacts", "Glob", "LazyArtifacts", "Libdl", "Pkg", "Printf", "RelocatableFolders", "TOML", "UUIDs", "p7zip_jll"]
git-tree-sha1 = "762cbdf85ab712e1b7aae75ae919f3a2c5a93896"
uuid = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
version = "2.1.14"

[[deps.PaddedViews]]
deps = ["OffsetArrays"]
git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f"
@@ -807,6 +818,12 @@ git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[deps.RelocatableFolders]]
deps = ["SHA", "Scratch"]
git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864"
uuid = "05181044-ff0b-4ac5-8273-598c1e38db00"
version = "1.0.1"

[[deps.Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
@@ -841,6 +858,12 @@ git-tree-sha1 = "81f7d934b52b2441f7b44520bd982fdb3607b0da"
uuid = "76ed43ae-9a5d-5a62-8c75-30186b810ce8"
version = "3.43.0+0"

[[deps.Scratch]]
deps = ["Dates"]
git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.2.1"

[[deps.SentinelArrays]]
deps = ["Dates", "Random"]
git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39"
1 change: 1 addition & 0 deletions omf/solvers/reopt_jl/REoptSolver/Project.toml
Original file line number Diff line number Diff line change
@@ -7,5 +7,6 @@ version = "0.1.0"
HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
PackageCompiler = "9b87118b-4619-50d2-8e1e-99f35a4d4d9d"
PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
REopt = "d36ad4e8-d74a-4f7a-ace1-eaea049febf6"
15 changes: 10 additions & 5 deletions omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl
Original file line number Diff line number Diff line change
@@ -42,26 +42,31 @@ end


#currently available solvers: SCIP, HiGHS
function run(json_path::String, reopt_inputs_path::String, output_path::String, outage_path::Union{Nothing,String}=nothing,
solver::String="SCIP", microgrid_only::Bool=false, max_runtime_s::Union{Nothing,Int}=nothing)
function run(path::String, outages::Bool=false, solver::String="SCIP", microgrid_only::Bool=false, max_runtime_s::Union{Nothing,Int}=nothing)

m = get_model(solver, max_runtime_s)
m2 = get_model(solver, max_runtime_s)
input_path = path * "/Scenario_test_POST.json"
reopt_inputs_path = path * "/REoptInputs.json"
output_path = path * "/results.json"

#writing REoptInputs to JSON for easier access of default values in microgridDesign
reopt_inputs = REoptInputs(json_path)
reopt_inputs = REoptInputs(input_path)
results_to_json(reopt_inputs,reopt_inputs_path)

results = run_reopt([m,m2],json_path)
results = run_reopt([m,m2],input_path)

results_to_json(results, output_path)

if outage_path != nothing
if outages != false
outage_path = path * "/resultsResilience.json"
outage_results = simulate_outages(results, reopt_inputs; microgrid_only=microgrid_only)

results_to_json(outage_results,outage_path)
end

end

precompile(run, (String, String, String, Union{Nothing,String}, String, Bool, Union{Nothing,String},))

end
119 changes: 36 additions & 83 deletions omf/solvers/reopt_jl/__init__.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,15 @@
import json, time
import os, platform

#html output visuals
#import test_outputs
#from omf.solvers.reopt_jl import test_outputs

thisDir = os.path.abspath(os.path.dirname(__file__))

#not currently working : REopt dependency (ArchGDAL) unable to precompile
#def build_julia_image():
# os.system(f'''julia --project={thisDir}/REoptSolver -e '
# import Pkg; Pkg.add("PackageCompiler");
# using PackageCompiler; include("{thisDir}/REoptSolver/src/REoptSolver.jl");
# PackageCompiler.precompile("REoptSolver")
# ' ''')

def build_julia_image():
os.system(f'''julia --project={thisDir}/REoptSolver -e '
import Pkg; import REoptSolver; using PackageCompiler;
PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{thisDir}/reopt_jl.so")
' ''')

#potential add: boolean to determine if you want to check your install
# => improve runtime if running multiple times in a row
def install_reopt_jl(system : list = platform.system()):
if os.path.isfile(f'{thisDir}/instantiated.txt'):
print("REopt.jl dependencies installed - to reinstall remove file: instantiated.txt")
@@ -45,7 +37,7 @@ def install_reopt_jl(system : list = platform.system()):

for command in commands:
os.system(command)
#build_julia_image()
build_julia_image()

########################################################
#functions for converting REopt input to REopt.jl input
@@ -208,8 +200,7 @@ def convert_to_jl(reopt_json):
print(e)
return new_jl_json

# for accessing & setting names of input/output json files

# for reading and writing input/output json files
def get_json(inputPath):
with open(inputPath) as j:
inputJson = json.load(j)
@@ -221,120 +212,82 @@ def write_json(outputPath, jsonData):
with open(outputPath, "w") as j:
json.dump(jsonData, j)

#input and output file names used with reopt_jl solver
def get_file_names(path, inputFile, default, convert, outages, solver, solver_in_filename):
inFile = inputFile if not default else "julia_default.json"
#input path to file chosen by user : default file found in current directory
inputPath = f'{path}/{inFile}' if not default else f'{thisDir}/{inFile}'
#path given to julia solver
jlInPath = f'{path}/converted_{inputFile}' if convert and not default else inputPath

REoptInputsPath = f'{path}/REoptInputs.json'

outFile = f'{solver}_{inFile}' if solver_in_filename else inFile
#path for output file
outputPath = f'{path}/out_{outFile}'
#path for output outage file if simulating outages
outagePath = f'{path}/outages_{outFile}' if outages else None

return (inputPath, jlInPath, REoptInputsPath, outputPath, outagePath)

##########################################################################
# run_reopt_jl : calls 'run' function through run_reopt.jl (Julia file)
##########################################################################

#todo: add options to set output path (and outage output path) ?
#potential optional inputs (for solver): ratio_gap, threads, max_solutions, verbosity
def run_reopt_jl(path, inputFile="", default=False, convert=True, outages=False, microgrid_only=False,
solver="HiGHS", solver_in_filename=True, max_runtime_s=None):
def run_reopt_jl(path, inputFile="", default=False, convert=False, outages=False, microgrid_only=False,
solver="HiGHS", max_runtime_s=None):

if inputFile == "" and not default:
print("Invalid inputs: inputFile needed if default=False")
return

install_reopt_jl()

file_info = (path, inputFile, default, convert, outages, solver, solver_in_filename)
(inPath, jlInPath, REoptInputsPath, outPath, outagePath) = get_file_names( *file_info )
constant_file = "Scenario_test_POST.json"
constant_path = f'{path}/{constant_file}'

if inputFile != constant_file:
inputPath = f'{path}/{inputFile}'
if default == True:
inputPath = f'{thisDir}/julia_default.json'
input_json = get_json(inputPath)
write_json(constant_path, input_json)

try:
if convert and not default: #default file is already converted
input_json = get_json(inPath)
input_json = get_json(constant_path)
reopt_jl_input_json = convert_to_jl(input_json)
write_json(jlInPath, reopt_jl_input_json)

from julia import Pkg #Julia, Main
#Julia(runtime=f'{thisDir}/reopt_jl.so')
#from julia import REoptSolver
Pkg.activate(f'{thisDir}/REoptSolver')
from julia import REoptSolver

REoptSolver.run(jlInPath, REoptInputsPath, outPath, outagePath, solver, microgrid_only, max_runtime_s)
#Main.include(f'{thisDir}/REoptSolver/src/REoptSolver.jl')
#Main.run(jlInPath, outPath, outagePath, solver, microgrid_only, max_runtime_s)
#todo: return output & outage path(s)? (decide on usage within models)
write_json(constant_path, reopt_jl_input_json)

microgrid_only_conv = "false" if not microgrid_only else "true"
outages_conv = "false" if not outages else "true"
max_runtime_s_conv = "nothing" if max_runtime_s == None else max_runtime_s

os.system(f'''julia --sysimage={f'{thisDir}/reopt_jl.so'} -e '
using .REoptSolver;
REoptSolver.run("{path}", {outages_conv}, "{solver}", {microgrid_only_conv}, {max_runtime_s_conv})
' ''')
except Exception as e:
print(e)

###########################################################################
# comparing REopt.jl outputs for different test cases / solvers
###########################################################################

def runAllSolvers(path, testName, fileName="", default=False, convert=True, outages=True,
solvers=["SCIP","HiGHS"], solver_in_filename=True, max_runtime_s=None,
get_cached=True ):
test_results = []
def runAllSolvers(path, testName, fileName="", default=False, convert=True, outages=True, solvers=["SCIP","HiGHS"], max_runtime_s=None):

for solver in solvers:
print(f'########## Running {solver} test: {testName}')
start = time.time()

file_info = (path, fileName, default, convert, outages, solver, solver_in_filename)
(_, _, outPath, outagePath) = get_file_names( *file_info )

if get_cached and os.path.isfile(outPath):
print("this test was already run: loading cached results")
else:
run_reopt_jl(path, inputFile=fileName, default=default, solver=solver, outages=outages,
max_runtime_s=max_runtime_s)
run_reopt_jl(path, inputFile=fileName, default=default, convert=convert, solver=solver, outages=outages, max_runtime_s=max_runtime_s)

end = time.time()
runtime = end - start
test_results.append((outPath, outagePath, testName, runtime, solver, outages, get_cached))
print(f'########## Completed {solver} test: {testName}')

return(test_results)
print(f'########## Completed {solver} test: {testName} in {runtime} seconds')


def _test():
all_tests = []
all_solvers = [ "HiGHS" ] # "Ipopt", "ECOS", "Clp", "GLPK", "SCIP", "Cbc"
path = f'{thisDir}/testFiles'

########### CONWAY_MG_MAX:
# CONWAY_MG.json copied from CONWAY_MG_MAX/Scenario_test_POST.json
CONWAY_tests = runAllSolvers(path, "CONWAY_MG_MAX", fileName="CONWAY_MG.json", solvers=all_solvers,
get_cached=False)
all_tests.extend(CONWAY_tests)
#runAllSolvers(path, "CONWAY_MG_MAX", fileName="CONWAY_MG.json", solvers=all_solvers)

############### CE test case
# CE.json copied from CE Test Case/Scenario_test_POST.json
CE_tests = runAllSolvers(path, "CE Test Case", fileName="CE.json", solvers=all_solvers,
get_cached=False, max_runtime_s=240)
all_tests.extend(CE_tests)
#runAllSolvers(path, "CE Test Case", fileName="CE.json", solvers=all_solvers, max_runtime_s=240)

############## CONWAY_30MAY23_SOLARBATTERY
# CONWAY_SB.json copied from CONWAY_30MAY23_SOLARBATTERY/Scenario_test_POST.json
CONWAY_SB_tests = runAllSolvers(path, "CONWAY_30MAY23_SOLARBATTERY", fileName="CONWAY_SB.json",
solvers=all_solvers, get_cached=False)
all_tests.extend(CONWAY_SB_tests)
#runAllSolvers(path, "CONWAY_30MAY23_SOLARBATTERY", fileName="CONWAY_SB.json", solvers=all_solvers)

####### default julia json (default values from microgridDesign)
default_tests = runAllSolvers(path, "Julia Default", default=True, solvers=all_solvers,
get_cached=False)
all_tests.extend(default_tests)

#test_outputs.html_comparison(all_tests) # => test_outputs.py (work in progress)
#runAllSolvers(path, "Julia Default", default=True, solvers=all_solvers)

if __name__ == "__main__":
_test()

0 comments on commit 4ebb87f

Please sign in to comment.