From 50737bf2d176c33bdfe593267da9588380628efb Mon Sep 17 00:00:00 2001 From: Lily Olson Date: Fri, 16 Feb 2024 11:38:14 -0500 Subject: [PATCH] reopt_jl: updates for filepath safety, updates to README --- omf/solvers/reopt_jl/README.md | 43 +++------ .../reopt_jl/REoptSolver/src/REoptSolver.jl | 9 +- omf/solvers/reopt_jl/__init__.py | 93 +++++++++++-------- omf/solvers/reopt_jl/precompile_reopt.jl | 11 ++- 4 files changed, 77 insertions(+), 79 deletions(-) diff --git a/omf/solvers/reopt_jl/README.md b/omf/solvers/reopt_jl/README.md index 9011d1484..5f8c9ef1b 100644 --- a/omf/solvers/reopt_jl/README.md +++ b/omf/solvers/reopt_jl/README.md @@ -3,21 +3,21 @@ omf.solvers.reopt_jl Solver for Julia verison of REopt. # Dependencies: -- python@3.11 (other versions of python3 may work) +- python@3.11.7 (other versions may require you to downgrade PyCall) - packages installed in install_reopt_jl() ( runs automatically within run_reopt_jl ) - julia (1.9.4) - PyJulia (0.6.1) -- packages within REoptSolver/Project.toml ( should be installed by Project.toml & Manifest.tomnl ) - - REopt@0.32.7 - - JuMP@1.13.0 - - HiGHS@1.7.2 +- packages within REoptSolver/Project.toml ( installed by Project.toml & Manifest.tomnl ) + - REopt@0.39.0 + - JuMP@1.17.0 + - HiGHS@1.7.5 - JSON (0.21.4) - - PyCall (1.96.1) - - PackageCompiler (2.1.10) + - PyCall (1.96.4) + - PackageCompiler (2.1.15) # Building REoptSolver Julia module: (avoid doing this unless the package change is necessary for REoptSolver to run) -* Project.toml & Mainfest.toml are included in /REoptSolver but can be modified with the following: +* Project.toml & Manifest.toml are included in /REoptSolver but can be modified with the following: ``` ~/omf/omf/solvers/reopt_jl % julia julia> ] @@ -32,21 +32,24 @@ julia> ] # Usage: ``` __init__.py: --> run_reopt_jl(path, inputFile="", default=False, convert=True, outages=False, microgrid_only=False, max_runtime_s=None) +-> run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None, run_with_sysimage=True) ``` General paramters: - path: directory containing inputFile ; output files written here as well - inputFile: json file containing REopt API input information - if this file is not converted for REopt.jl -> set convert=True +- loadFile: csv load file for the given input file + - if empty: assumes that the csv load path within inputFile is already set + - otherwise: the path is set to path/loadFile - outages: if True, runs outage simulation, otherwise doesn't - microgrid_only: if True runs without grid, otherwise runs as normal *only used within REopt.jl currently (not API) - max_runtime_s: default is None, otherwise times out after given number of seconds and returns local optimal value (may not be the global optimum) +- run_with_sysimage: if True, runs with reopt_jl.so (builds beforehand if necessary), otherwise runs by loading REoptSolver project Testing parameters: - default: if True, sets inputFile to default values found in julia_default.json, uses given inputFile otherwise -- convert: if True, converts variables names to those used in REopt.jl, no conversion otherwise Examples: ``` @@ -64,23 +67,3 @@ uses julia_default.json as input and writes ouput file from REopt.jl to "current ``` writes ouput file from REopt.jl to "currentDir/results.json" and writes outage output fie to "currentDir/resultsResilience.json" - -# Testing usage: -``` -__init__.py: --> runAllSolvers(path, testName, fileName="", default=False, convert=True, outages=True, solvers=["SCIP","HiGHS"], max_runtime_s=None, get_cached=True ) -``` - -Usage: simlar to run_reopt_jl but takes in list of solvers and runs each one on the given test case ; prints out runtime comparisons -- The list of available solvers is currently limited to HiGHS to reduce compile time - -Inputs: -- path : run_reopt_jl path -- testName : name used to identify test case -- fileName : run_reopt_jl inputFile -- default : run_reopt_jl default -- convert : run_reopt_jl convert -- outages : run_reopt_jl outages -- solvers : list of solvers to call run_reopt_jl with - -test_outputs.py => DEPRECATED (can be found in REopt_replacements in wiires repository but needs updates to function) diff --git a/omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl b/omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl index d6967e091..b142a6ff3 100644 --- a/omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl +++ b/omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl @@ -2,6 +2,7 @@ module REoptSolver export run +using Base.Filesystem #all solvers except for HiGHS are currently removed in order to improve precompile and load time using REopt, JuMP, JSON, HiGHS #SCIP #, Cbc #Ipopt, ECOS, Clp, GLPK @@ -30,9 +31,9 @@ function run(path::String, outages::Bool=false, microgrid_only::Bool=false, max_ m = get_model(path, max_runtime_s) m2 = get_model(path, max_runtime_s) - input_path = path * "/Scenario_test_POST.json" - reopt_inputs_path = path * "/REoptInputs.json" - output_path = path * "/results.json" + input_path = normpath(joinpath(path, "Scenario_test_POST.json")) + reopt_inputs_path = normpath(joinpath(path,"REoptInputs.json")) + output_path = normpath(joinpath(path,"results.json")) #writing REoptInputs to JSON for easier access of default values in microgridDesign reopt_inputs = REoptInputs(input_path) @@ -43,7 +44,7 @@ function run(path::String, outages::Bool=false, microgrid_only::Bool=false, max_ results_to_json(results, output_path) if outages != false - outage_path = path * "/resultsResilience.json" + outage_path = normpath(joinpath(path,"resultsResilience.json")) outage_results = simulate_outages(results, reopt_inputs; microgrid_only=microgrid_only) results_to_json(outage_results,outage_path) diff --git a/omf/solvers/reopt_jl/__init__.py b/omf/solvers/reopt_jl/__init__.py index 622b6c9d4..5fe467552 100644 --- a/omf/solvers/reopt_jl/__init__.py +++ b/omf/solvers/reopt_jl/__init__.py @@ -2,19 +2,18 @@ import os, platform import random -thisDir = os.path.abspath(os.path.dirname(__file__)) +thisDir = str(os.path.abspath(os.path.dirname(__file__))) -#potential add: boolean to determine if you want to check your install -def install_reopt_jl(system : list = platform.system()): +def install_reopt_jl(system : list = platform.system(), build_sysimage=True): ''' Installs dependencies necessary for running REoptSolver and creates sysimage to reduce precompile time ''' - instantiated_path = os.path.join(thisDir,"instantiated.txt") - project_path = os.path.join(thisDir,"REoptSolver") - sysimage_path = os.path.join(thisDir,"reopt_jl.so") - precompile_path = os.path.join(thisDir,"precompile_reopt.jl") + instantiated_path = str(os.path.normpath(os.path.join(thisDir,"instantiated.txt"))) + project_path = str(os.path.normpath(os.path.join(thisDir,"REoptSolver"))) + sysimage_path = str(os.path.normpath(os.path.join(thisDir,"reopt_jl.so"))) + precompile_path = str(os.path.normpath(os.path.join(thisDir,"precompile_reopt.jl"))) if os.path.isfile(instantiated_path): print("reopt_jl dependencies installed - to reinstall remove file: instantiated.txt") - if not os.path.isfile(sysimage_path): + if not os.path.isfile(sysimage_path) and build_sysimage: print("error: reopt_jl sysimage not found - remove instantiated.txt to build") return @@ -24,20 +23,22 @@ def install_reopt_jl(system : list = platform.system()): '''python3 -c 'import julia; julia.install()' ''' ] #adding abilty to use pre-made sysimage (not instantiated) - if os.path.exists(sysimage_path): + if os.path.exists(sysimage_path) and build_sysimage: print("pre-built reopt_jl.so sysimage found") build_julia_image = [ - f'chmod +x {sysimage_path}' + f'chmod +x "{sysimage_path}"' ] - else: + elif build_sysimage: build_julia_image = [ - f'''julia --project={project_path} -e ' + f'''julia --project="{project_path}" -e ' import Pkg; Pkg.instantiate(); import REoptSolver; using PackageCompiler; PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)") ' ''' ] + else: + build_julia_image = [] if system == "Darwin": commands = [ ''' HOMEBREW_NO_AUTO_UPDATE=1 brew list julia 1>/dev/null 2>/dev/null || @@ -46,7 +47,7 @@ def install_reopt_jl(system : list = platform.system()): ] commands += install_pyjulia commands += build_julia_image - commands += [ f'touch {instantiated_path}' ] + commands += [ f'touch "{instantiated_path}"' ] elif system == "Linux": commands = [ 'sudo apt-get install wget', @@ -56,22 +57,23 @@ def install_reopt_jl(system : list = platform.system()): ] commands += install_pyjulia commands += build_julia_image - commands += [ f'touch {instantiated_path}' ] + commands += [ f'touch "{instantiated_path}"' ] elif system == "Windows": commands = [ - f'cd {thisDir} & del julia-1.9.4-win64.zip', - f'cd {thisDir} & curl -o julia-1.9.4-win64.zip https://julialang-s3.julialang.org/bin/winnt/x64/1.9/julia-1.9.4-win64.zip', - f'cd {thisDir} & tar -x -f "julia-1.9.4-win64.zip' ] + f'cd "{thisDir}" & del julia-1.9.4-win64.zip', + f'cd "{thisDir}" & curl -o julia-1.9.4-win64.zip https://julialang-s3.julialang.org/bin/winnt/x64/1.9/julia-1.9.4-win64.zip', + f'cd "{thisDir}" & tar -x -f "julia-1.9.4-win64.zip' ] commands += install_pyjulia - commands += [ - f'''cd {thisDir}\\julia-1.9.4\\bin & julia --project={project_path} -e ' - import Pkg; Pkg.instantiate(); - import REoptSolver; using PackageCompiler; - PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", - precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)") - ' ''' - f'copy nul {thisDir}\\instantiated.txt' - ] + if build_sysimage: + commands += [ + f'''cd "{thisDir}\\julia-1.9.4\\bin" & julia --project="{project_path}" -e ' + import Pkg; Pkg.instantiate(); + import REoptSolver; using PackageCompiler; + PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", + precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)") + ' ''' + f'copy nul "{instantiated_path}"' + ] else: raise ValueError(f'No installation script available yet for {system}') @@ -250,27 +252,29 @@ def get_randomized_api_key(): #potential optional inputs (for solver): ratio_gap, threads, max_solutions, verbosity #potential other inputs (ease of use): load csv file path ("path_to_csv") => check if "loads_kw" already exists -def run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None): +def run_reopt_jl(path, inputFile="", loadFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None, + run_with_sysimage=True ): ''' calls 'run' function through run_reopt.jl (Julia file) ''' if inputFile == "" and not default: print("Invalid inputs: inputFile needed if default=False") return - install_reopt_jl() + install_reopt_jl(build_sysimage=run_with_sysimage) constant_file = "Scenario_test_POST.json" - constant_path = os.path.join(path,constant_file) + constant_path = os.path.normpath(os.path.join(path,constant_file)) if inputFile != constant_file: - inputPath = os.path.join(path,inputFile) + inputPath = os.path.normpath(os.path.join(path,inputFile)) if default == True: - inputPath = os.path.join(thisDir,"testFiles","julia_default.json") + inputPath = os.path.normpath(os.path.join(thisDir,"testFiles","julia_default.json")) with open(inputPath) as j: input_json = json.load(j) - #potential helper function: add_load_path => sets file path in json based on current directory if default == True: - input_json["ElectricLoad"]["path_to_csv"] = os.path.join(thisDir,"testFiles","loadShape.csv") + input_json["ElectricLoad"]["path_to_csv"] = os.path.normpath(os.path.join(thisDir,"testFiles","loadShape.csv")) + elif loadFile != "": + input_json["ElectricLoad"]["path_to_csv"] = os.path.normpath(os.path.join(path,loadFile)) with open(constant_path, "w") as j: json.dump(input_json, j) @@ -280,19 +284,28 @@ def run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_onl max_runtime_s_jl = "nothing" if max_runtime_s == None else max_runtime_s api_key = get_randomized_api_key() - sysimage_path = os.path.join(thisDir,"reopt_jl.so") - os.system(f'''julia --sysimage={sysimage_path} -e ' - using .REoptSolver; - ENV["NREL_DEVELOPER_API_KEY"]="{api_key}"; - REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}") - ' ''') + if run_with_sysimage: + sysimage_path = os.path.normpath(os.path.join(thisDir,"reopt_jl.so")) + os.system(f'''julia --sysimage="{sysimage_path}" -e ' + using .REoptSolver; + ENV["NREL_DEVELOPER_API_KEY"]="{api_key}"; + REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}") + ' ''') + else: + project_path = os.path.normpath(os.path.join(thisDir,"REoptSolver")) + os.system(f'''julia --project="{project_path}" -e ' + using Pkg; Pkg.instantiate(); + import REoptSolver; + ENV["NREL_DEVELOPER_API_KEY"]="{api_key}"; + REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}") + ' ''') except Exception as e: print(e) def _test(): - run_reopt_jl(os.path.join(thisDir,"testFiles"), default=True) + run_reopt_jl(os.path.normpath(os.path.join(thisDir,"testFiles")), default=True) if __name__ == "__main__": _test() \ No newline at end of file diff --git a/omf/solvers/reopt_jl/precompile_reopt.jl b/omf/solvers/reopt_jl/precompile_reopt.jl index d553d1081..d56e1f8c3 100644 --- a/omf/solvers/reopt_jl/precompile_reopt.jl +++ b/omf/solvers/reopt_jl/precompile_reopt.jl @@ -1,15 +1,16 @@ using JSON +using Base.Filesystem -include("REoptSolver/src/REoptSolver.jl") #joinpath(@__DIR__, +include(normpath(joinpath("REoptSolver","src","REoptSolver.jl"))) #joinpath(@__DIR__, function test() - json_data = JSON.parsefile(joinpath(@__DIR__, "julia_default.json")) - json_data["ElectricLoad"]["path_to_csv"] = joinpath(@__DIR__,"testFiles/loadShape.csv") + json_data = JSON.parsefile(normpath(joinpath(string(@__DIR__), "testFiles","julia_default.json"))) + json_data["ElectricLoad"]["path_to_csv"] = normpath(joinpath(string(@__DIR__),"testFiles","loadShape.csv")) - open(joinpath(@__DIR__, "testFiles/Scenario_test_POST.json"), "w") do file + open(normpath(joinpath(string(@__DIR__), "testFiles","Scenario_test_POST.json")), "w") do file JSON.print(file, json_data) end - REoptSolver.run(joinpath(@__DIR__,"testFiles"), true) + REoptSolver.run(normpath(joinpath(string(@__DIR__),"testFiles")), true) end test() \ No newline at end of file