reopt_jl: updates for filepath safety, updates to README

omf/solvers/reopt_jl/README.md

@@ -3,21 +3,21 @@ omf.solvers.reopt_jl
 Solver for Julia verison of REopt. 
 
 # Dependencies:
-- [email protected] (other versions of python3 may work)
+- [email protected].7 (other versions may require you to downgrade PyCall)
 - packages installed in install_reopt_jl()  ( runs automatically within run_reopt_jl )
     - julia (1.9.4)
     - PyJulia (0.6.1)
-- packages within REoptSolver/Project.toml ( should be installed by Project.toml & Manifest.tomnl )
-    - REopt@0.32.7
-    - JuMP@1.13.0
-    - [email protected].2
+- packages within REoptSolver/Project.toml ( installed by Project.toml & Manifest.tomnl )
+    - REopt@0.39.0
+    - JuMP@1.17.0
+    - [email protected].5
     - JSON (0.21.4)
-    - PyCall (1.96.1)
-    - PackageCompiler (2.1.10)
+    - PyCall (1.96.4)
+    - PackageCompiler (2.1.15)
 
 # Building REoptSolver Julia module:
 (avoid doing this unless the package change is necessary for REoptSolver to run)
-* Project.toml & Mainfest.toml are included in /REoptSolver but can be modified with the following:
+* Project.toml & Manifest.toml are included in /REoptSolver but can be modified with the following:
 ```
 ~/omf/omf/solvers/reopt_jl % julia
 julia> ]
@@ -32,21 +32,24 @@ julia> ]
 # Usage:
 ```
 __init__.py:
--> run_reopt_jl(path, inputFile="", default=False, convert=True, outages=False, microgrid_only=False, max_runtime_s=None)
+-> run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None, run_with_sysimage=True)
 ```
 
 General paramters:
 - path: directory containing inputFile ; output files written here as well
 - inputFile: json file containing REopt API input information
     - if this file is not converted for REopt.jl -> set convert=True
+- loadFile: csv load file for the given input file
+    - if empty: assumes that the csv load path within inputFile is already set
+    - otherwise: the path is set to path/loadFile 
 - outages: if True, runs outage simulation, otherwise doesn't
 - microgrid_only: if True runs without grid, otherwise runs as normal
     *only used within REopt.jl currently (not API)
 - max_runtime_s: default is None, otherwise times out after given number of seconds and returns local optimal value (may not be the global optimum)
+- run_with_sysimage: if True, runs with reopt_jl.so (builds beforehand if necessary), otherwise runs by loading REoptSolver project
 
 Testing parameters:
 - default: if True, sets inputFile to default values found in julia_default.json, uses given inputFile otherwise
-- convert: if True, converts variables names to those used in REopt.jl, no conversion otherwise
 
 Examples:
 ``` 
@@ -64,23 +67,3 @@ uses julia_default.json as input and writes ouput file from REopt.jl to "current
 ```
 writes ouput file from REopt.jl to "currentDir/results.json" and
 writes outage output fie to "currentDir/resultsResilience.json"
-
-# Testing usage:
-```
-__init__.py: 
--> runAllSolvers(path, testName, fileName="", default=False, convert=True, outages=True, solvers=["SCIP","HiGHS"], max_runtime_s=None, get_cached=True )
-```
-
-Usage: simlar to run_reopt_jl but takes in list of solvers and runs each one on the given test case ; prints out runtime comparisons
-- The list of available solvers is currently limited to HiGHS to reduce compile time
-
-Inputs:
-- path : run_reopt_jl path
-- testName : name used to identify test case
-- fileName : run_reopt_jl inputFile
-- default : run_reopt_jl default
-- convert : run_reopt_jl convert
-- outages : run_reopt_jl outages
-- solvers : list of solvers to call run_reopt_jl with
-
-test_outputs.py => DEPRECATED (can be found in REopt_replacements in wiires repository but needs updates to function)

omf/solvers/reopt_jl/REoptSolver/src/REoptSolver.jl

@@ -2,6 +2,7 @@ module REoptSolver
 
 export run
 
+using Base.Filesystem
 #all solvers except for HiGHS are currently removed in order to improve precompile and load time
 using REopt, JuMP, JSON, HiGHS #SCIP #, Cbc
 #Ipopt, ECOS, Clp, GLPK
@@ -30,9 +31,9 @@ function run(path::String, outages::Bool=false, microgrid_only::Bool=false, max_
 
 	m = get_model(path, max_runtime_s)
 	m2 = get_model(path, max_runtime_s)
-	input_path = path * "/Scenario_test_POST.json"
-	reopt_inputs_path = path * "/REoptInputs.json"
-	output_path = path * "/results.json"
+	input_path = normpath(joinpath(path, "Scenario_test_POST.json"))
+	reopt_inputs_path = normpath(joinpath(path,"REoptInputs.json"))
+	output_path = normpath(joinpath(path,"results.json"))
 
 	#writing REoptInputs to JSON for easier access of default values in microgridDesign
 	reopt_inputs = REoptInputs(input_path)
@@ -43,7 +44,7 @@ function run(path::String, outages::Bool=false, microgrid_only::Bool=false, max_
 	results_to_json(results, output_path)
 
 	if outages != false
-		outage_path = path * "/resultsResilience.json"
+		outage_path = normpath(joinpath(path,"resultsResilience.json"))
 		outage_results = simulate_outages(results, reopt_inputs; microgrid_only=microgrid_only)
 
 		results_to_json(outage_results,outage_path)

omf/solvers/reopt_jl/__init__.py

@@ -2,19 +2,18 @@
 import os, platform
 import random
 
-thisDir = os.path.abspath(os.path.dirname(__file__))
+thisDir = str(os.path.abspath(os.path.dirname(__file__)))
 
-#potential add: boolean to determine if you want to check your install
-def install_reopt_jl(system : list = platform.system()):
+def install_reopt_jl(system : list = platform.system(), build_sysimage=True):
     ''' Installs dependencies necessary for running REoptSolver and creates sysimage to reduce precompile time '''
-    instantiated_path = os.path.join(thisDir,"instantiated.txt")
-    project_path = os.path.join(thisDir,"REoptSolver")
-    sysimage_path = os.path.join(thisDir,"reopt_jl.so")
-    precompile_path = os.path.join(thisDir,"precompile_reopt.jl")
+    instantiated_path = str(os.path.normpath(os.path.join(thisDir,"instantiated.txt")))
+    project_path = str(os.path.normpath(os.path.join(thisDir,"REoptSolver")))
+    sysimage_path = str(os.path.normpath(os.path.join(thisDir,"reopt_jl.so")))
+    precompile_path = str(os.path.normpath(os.path.join(thisDir,"precompile_reopt.jl")))
 
     if os.path.isfile(instantiated_path):
         print("reopt_jl dependencies installed - to reinstall remove file: instantiated.txt")
-        if not os.path.isfile(sysimage_path):
+        if not os.path.isfile(sysimage_path) and build_sysimage:
             print("error: reopt_jl sysimage not found - remove instantiated.txt to build")
         return
 
@@ -24,20 +23,22 @@ def install_reopt_jl(system : list = platform.system()):
                 '''python3 -c 'import julia; julia.install()' '''
         ]
         #adding abilty to use pre-made sysimage (not instantiated)
-        if os.path.exists(sysimage_path):
+        if os.path.exists(sysimage_path) and build_sysimage:
             print("pre-built reopt_jl.so sysimage found")
             build_julia_image = [
-                f'chmod +x {sysimage_path}'
+                f'chmod +x "{sysimage_path}"'
             ]
-        else:
+        elif build_sysimage:
             build_julia_image = [
-                f'''julia --project={project_path} -e '
+                f'''julia --project="{project_path}" -e '
                 import Pkg; Pkg.instantiate();
                 import REoptSolver; using PackageCompiler;
                 PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", 
                 precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)")
                 ' '''
             ]
+        else:
+            build_julia_image = []
         if system == "Darwin":
             commands = [ '''
                 HOMEBREW_NO_AUTO_UPDATE=1 brew list julia 1>/dev/null 2>/dev/null || 
@@ -46,7 +47,7 @@ def install_reopt_jl(system : list = platform.system()):
             ]
             commands += install_pyjulia
             commands += build_julia_image
-            commands += [ f'touch {instantiated_path}' ]
+            commands += [ f'touch "{instantiated_path}"' ]
         elif system == "Linux":
             commands = [
                 'sudo apt-get install wget',
@@ -56,22 +57,23 @@ def install_reopt_jl(system : list = platform.system()):
             ]
             commands += install_pyjulia
             commands += build_julia_image
-            commands += [ f'touch {instantiated_path}' ]
+            commands += [ f'touch "{instantiated_path}"' ]
         elif system == "Windows":
             commands = [
-                f'cd {thisDir} & del julia-1.9.4-win64.zip',
-			    f'cd {thisDir} & curl -o julia-1.9.4-win64.zip https://julialang-s3.julialang.org/bin/winnt/x64/1.9/julia-1.9.4-win64.zip',
-			    f'cd {thisDir} & tar -x -f "julia-1.9.4-win64.zip' ]
+                f'cd "{thisDir}" & del julia-1.9.4-win64.zip',
+			    f'cd "{thisDir}" & curl -o julia-1.9.4-win64.zip https://julialang-s3.julialang.org/bin/winnt/x64/1.9/julia-1.9.4-win64.zip',
+			    f'cd "{thisDir}" & tar -x -f "julia-1.9.4-win64.zip' ]
             commands += install_pyjulia
-            commands += [
-                f'''cd {thisDir}\\julia-1.9.4\\bin & julia --project={project_path} -e '
-                import Pkg; Pkg.instantiate();
-                import REoptSolver; using PackageCompiler;
-                PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", 
-                precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)")
-                ' '''
-			    f'copy nul {thisDir}\\instantiated.txt'
-            ]
+            if build_sysimage:
+                commands += [
+                    f'''cd "{thisDir}\\julia-1.9.4\\bin" & julia --project="{project_path}" -e '
+                    import Pkg; Pkg.instantiate();
+                    import REoptSolver; using PackageCompiler;
+                    PackageCompiler.create_sysimage(["REoptSolver"]; sysimage_path="{sysimage_path}", 
+                    precompile_execution_file="{precompile_path}", cpu_target="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)")
+                    ' '''
+			        f'copy nul "{instantiated_path}"'
+                ]
         else:
             raise ValueError(f'No installation script available yet for {system}')
 
@@ -250,27 +252,29 @@ def get_randomized_api_key():
 
 #potential optional inputs (for solver): ratio_gap, threads, max_solutions, verbosity
 #potential other inputs (ease of use): load csv file path ("path_to_csv") => check if "loads_kw" already exists
-def run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None):
+def run_reopt_jl(path, inputFile="", loadFile="", default=False, outages=False, microgrid_only=False, max_runtime_s=None, 
+                 run_with_sysimage=True ):
     ''' calls 'run' function through run_reopt.jl (Julia file) '''
 
     if inputFile == "" and not default:
         print("Invalid inputs: inputFile needed if default=False")
         return
 
-    install_reopt_jl()
+    install_reopt_jl(build_sysimage=run_with_sysimage)
 
     constant_file = "Scenario_test_POST.json"
-    constant_path = os.path.join(path,constant_file)
+    constant_path = os.path.normpath(os.path.join(path,constant_file))
 
     if inputFile != constant_file:
-        inputPath = os.path.join(path,inputFile)
+        inputPath = os.path.normpath(os.path.join(path,inputFile))
         if default == True:
-            inputPath = os.path.join(thisDir,"testFiles","julia_default.json")
+            inputPath = os.path.normpath(os.path.join(thisDir,"testFiles","julia_default.json"))
         with open(inputPath) as j:
             input_json = json.load(j)
-            #potential helper function: add_load_path => sets file path in json based on current directory
             if default == True:
-                input_json["ElectricLoad"]["path_to_csv"] = os.path.join(thisDir,"testFiles","loadShape.csv")
+                input_json["ElectricLoad"]["path_to_csv"] = os.path.normpath(os.path.join(thisDir,"testFiles","loadShape.csv"))
+            elif loadFile != "":
+                input_json["ElectricLoad"]["path_to_csv"] = os.path.normpath(os.path.join(path,loadFile))
             with open(constant_path, "w") as j:
                 json.dump(input_json, j)
 
@@ -280,19 +284,28 @@ def run_reopt_jl(path, inputFile="", default=False, outages=False, microgrid_onl
         max_runtime_s_jl = "nothing" if max_runtime_s == None else max_runtime_s
 
         api_key = get_randomized_api_key()
-        sysimage_path = os.path.join(thisDir,"reopt_jl.so")
 
-        os.system(f'''julia --sysimage={sysimage_path} -e '
-                  using .REoptSolver;
-                  ENV["NREL_DEVELOPER_API_KEY"]="{api_key}";
-                  REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}")
-                  ' ''')
+        if run_with_sysimage:
+            sysimage_path = os.path.normpath(os.path.join(thisDir,"reopt_jl.so"))
+            os.system(f'''julia --sysimage="{sysimage_path}" -e '
+                      using .REoptSolver;
+                      ENV["NREL_DEVELOPER_API_KEY"]="{api_key}";
+                      REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}")
+                      ' ''')
+        else:
+            project_path = os.path.normpath(os.path.join(thisDir,"REoptSolver"))
+            os.system(f'''julia --project="{project_path}" -e '
+                      using Pkg; Pkg.instantiate();
+                      import REoptSolver;
+                      ENV["NREL_DEVELOPER_API_KEY"]="{api_key}";
+                      REoptSolver.run("{path}", {outages_jl}, {microgrid_only_jl}, {max_runtime_s_jl}, "{api_key}")
+                      ' ''')
     except Exception as e:
         print(e)
 
 
 def _test():
-    run_reopt_jl(os.path.join(thisDir,"testFiles"), default=True)
+    run_reopt_jl(os.path.normpath(os.path.join(thisDir,"testFiles")), default=True)
 
 if __name__ == "__main__":
     _test()

omf/solvers/reopt_jl/precompile_reopt.jl

@@ -1,15 +1,16 @@
 using JSON
+using Base.Filesystem
 
-include("REoptSolver/src/REoptSolver.jl") #joinpath(@__DIR__,
+include(normpath(joinpath("REoptSolver","src","REoptSolver.jl"))) #joinpath(@__DIR__,
 
 function test()
-	json_data = JSON.parsefile(joinpath(@__DIR__, "julia_default.json"))
-	json_data["ElectricLoad"]["path_to_csv"] = joinpath(@__DIR__,"testFiles/loadShape.csv")
+	json_data = JSON.parsefile(normpath(joinpath(string(@__DIR__), "testFiles","julia_default.json")))
+	json_data["ElectricLoad"]["path_to_csv"] = normpath(joinpath(string(@__DIR__),"testFiles","loadShape.csv"))
 
-    open(joinpath(@__DIR__, "testFiles/Scenario_test_POST.json"), "w") do file 
+    open(normpath(joinpath(string(@__DIR__), "testFiles","Scenario_test_POST.json")), "w") do file 
         JSON.print(file, json_data)
     end
-	REoptSolver.run(joinpath(@__DIR__,"testFiles"), true)
+	REoptSolver.run(normpath(joinpath(string(@__DIR__),"testFiles")), true)
 end
 
 test()