Added .ods and .csv as possible filetype inputs

AUTHORS

@@ -5,7 +5,7 @@
 # https://github.com/ramp-project/ramp/graphs/contributors.
 # Other collaborators and scientific advisors who did not necessarily contribute
 # to the Python code are featured in the peer-reviewed publications (https://rampdemand.org/publications/).
-# Moreover, contributors to the RAMP-mobility project are currently 
+# Moreover, contributors to the RAMP-mobility project are currently
 # listed separately at https://github.com/ramp-project/ramp-mobility.
 
 Francesco Lombardi, TU Delft <f.lombardi@tudelft.nl>
@@ -24,3 +24,4 @@ Francesco Sanvito, TU Delft <F.Sanvito@tudelft.nl>
 Gregory Ireland, Reiner Lemoine Institut <Gregory.Ireland@rl-institut.de>
 Sergio Balderrama, Universidad Mayor de San Simon <slbalderrama@fcyt.umss.edu.bo>
 Johann Kraft, Reiner Lemoine Institut <johann.kraft@rl-institut.de>
+Pedro Rene Lima Camacho, <pedrolc05@gmail.com>

CHANGELOG.md

@@ -4,6 +4,8 @@ Release History
 0.5.2 (dev)
 -----------
 
+**|new|**       Addition of .ods and .csv filetypes as inputs to CLI
+
 **|new|**       Addition of the 'coveralls' badge to the README
 
 **|new|**       Addition of a random-seed functionality to ensure reproducible results if needed

ramp/core/core.py

@@ -557,7 +557,7 @@ def export_to_dataframe(self) -> pd.DataFrame:
         return self.save()
 
     def load(self, filename: str) -> None:
-        """Open an .xlsx file which was produced via the save method and create instances of Users and Appliances
+        """Open an (.xlsx | .ods | .csv) file which was produced via the save method and create instances of Users and Appliances
 
         Parameters
         ----------

ramp/core/utils.py

@@ -2,23 +2,43 @@
 import random
 import time
 import datetime
+import re
+import os
 import numpy as np
 import pandas as pd
 from openpyxl import load_workbook
 from openpyxl.worksheet.cell_range import CellRange
+from odf.table import Table
+from odf.opendocument import load
 
 POSSIBLE_FORMATS = """
     The possible formats of the power timeseries are :
         - a single value (int or float) if the power is constant throughout the load_profile (with random fluctuations if the variable 'thermal_p_var' is provided)
-        - an array of value provided as text in a json array format, i.e. : [val1, val2, val3, ...]
+        - an array of value provided as text in a JSON array format, i.e. : [val1, val2, val3, ...]
         - a range of cells in another sheet of the input file (type '=' in the cell and then select the wished range of values to get the correct format automatically)
 
     ***Note***
-    The last two formats will only accept one column (values only) or two columns (timestamps and values, respectively) and exactly 365 rows
+    The last two formats will only accept one column (values only) or two columns (timestamps and values, respectively) and exactly 366 rows
+    CSV file only accepts JSON array format for power timeseries.
 """
 
 
 def read_input_file(filename):
+    """Parse a RAMP input file based on its type (XLSX, ODS, CSV)"""
+    _, file_extension = os.path.splitext(filename)
+    if file_extension == ".xlsx":
+        return read_excel_file(filename)
+    elif file_extension == ".ods":
+        return read_ods_file(filename)
+    elif file_extension == ".csv":
+        return read_csv_file(filename)
+    else:
+        raise ValueError(
+            "Unsupported file format. Please provide 'xlsx', 'ods', or 'csv'."
+        )
+
+
+def read_excel_file(filename):
     """Parse a RAMP .xlsx input file"""
 
     wb = load_workbook(filename=filename)
@@ -89,6 +109,109 @@ def read_input_file(filename):
     return df
 
 
+def read_ods_file(filename):
+    """Parse a RAMP .ods input file"""
+    df = pd.read_excel(filename, engine="odf")
+    doc = load(filename)
+
+    # df = pd.DataFrame(data[1:], columns=headers)  # Skip header row
+    df = df.fillna(value=np.nan)
+
+    df["p_series"] = False
+    for i, v in enumerate(df["power"].values):
+        appliance_name = df["name"].iloc[i]
+        user_name = df["user_name"].iloc[i]
+
+        if isinstance(v, str):
+            # the timeseries is provided as a range of values in the spreadsheet like "Sheet2!A2:B367"
+            if "[" not in v:
+                ts_sheet_name, ts_range = v.split(".")
+                read_excel_args = range_string_to_pandas_read_excel_args(ts_range)
+
+                if read_excel_args.get("nrows") != 366:
+                    raise ValueError(
+                        f"The provided range for the power timeseries of the appliance '{appliance_name}' of user '{user_name}' in '{filename}' does not contain 366 values as expected (range {ts_range} of sheet '{ts_sheet_name}')\n{POSSIBLE_FORMATS}"
+                    )
+                df_power = pd.read_excel(
+                    filename,
+                    engine="odf",
+                    sheet_name=ts_sheet_name,
+                    header=None,
+                    **read_excel_args,
+                )
+                if df_power.shape[1] == 1:
+                    ts = df_power.iloc[:, 0].to_json(orient="values")
+                    print(f"=====ts: {ts}")
+                elif df_power.shape[1] == 2:
+                    ts = df_power.to_json(orient="values")
+                else:
+                    raise ValueError(
+                        f"The provided range for the power timeseries of the appliance '{appliance_name}' of user '{user_name}' spans more than two columns in '{filename}' (range {ts_range} of sheet '{ts_sheet_name}')\n{POSSIBLE_FORMATS}"
+                    )
+            # the timeseries is expected as an array in json format
+            else:
+                try:
+                    ts = json.loads(v)
+                    if len(ts) != 366:
+                        raise ValueError(
+                            f"The provided power timeseries of the appliance '{appliance_name}' of user '{user_name}' in '{filename}' does not contain 366 values as expected\n{POSSIBLE_FORMATS}"
+                        )
+                    ts = v
+                except json.JSONDecodeError:
+                    raise ValueError(
+                        f"Could not parse the power timeseries provided for appliance '{appliance_name}' of user '{user_name}' in '{filename}'\n{POSSIBLE_FORMATS}"
+                    )
+            df.loc[i, "power"] = ts
+            df.loc[i, "p_series"] = True
+    return df
+
+
+def read_csv_file(filename):
+    df = pd.read_csv(filename)
+    df = df.fillna(value=np.nan)
+    df["p_series"] = False
+
+    def power_convertion(row):
+        value = row["power"]
+        try:
+            row["power"] = int(value)
+            return row
+        except (ValueError, TypeError):
+            try:
+                power_list = json.loads(value)
+            except:
+                raise ValueError(
+                    f"Could not parse the power timeseries provided in '{filename} at index {row.name}'\n{POSSIBLE_FORMATS}"
+                )
+
+            if len(power_list) != 366:
+                raise ValueError(
+                    f"The provided power timeseries does not contain 366 values as expected in '{filename} at index {row.name}'\n{POSSIBLE_FORMATS}"
+                )
+
+            row["p_series"] = True
+            return row
+
+    df = df.apply(power_convertion, axis=1)
+
+    return df
+
+
+def range_string_to_pandas_read_excel_args(cell_range):
+    """Converts a range of cells (like format 'A2:B367') and returns corresponding
+    arguments to use in pandas.read_excel method.
+    """
+    matches = re.findall(r"([A-Za-z]+)(\d+)", cell_range)
+    if len(matches) != 2:
+        raise ValueError(f"Expected 2 parts of cell range, got {len(matches)} parts.")
+
+    return {
+        "usecols": f"{matches[0][0]}:{matches[1][0]}",
+        "skiprows": int(matches[0][1]) - 1,
+        "nrows": int(matches[1][1]) - int(matches[0][1]) + 1,
+    }
+
+
 def random_variation(var, norm=1):
     """Pick a random variable within a uniform distribution of range [1-var, 1+var]
 

ramp/ramp_run.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 Created on Fri Apr 19 14:35:00 2019
-This is the code for the open-source stochastic model for the generation of 
+This is the code for the open-source stochastic model for the generation of
 multi-energy load profiles in off-grid areas, called RAMP, v0.3.0.
 
 @authors:
@@ -45,7 +45,7 @@ def run_usecase(
     plot=True,
     parallel=False,
 ):
-    if fname.endswith(".xlsx"):
+    if fname.endswith(".xlsx") or fname.endswith(".ods") or fname.endswith(".csv"):
         usecase = UseCase(
             date_start=date_start, date_end=date_end, parallel_processing=parallel
         )
@@ -66,5 +66,5 @@ def run_usecase(
             raise FileNotFoundError(f"{fname} is not an existing file")
     else:
         raise TypeError(
-            "Only the .py and .xlsx file format are supported for ramp command line"
+            "Only the .py | .xlsx | .ods | .csv file format are supported for ramp command line"
         )

tests/test_cli.py

@@ -47,7 +47,45 @@ def test_impossible_option_combinaison_end_date_year(self, m_args):
             ]
         ),
     )
-    def test_multiple_year_is_possible(self, m_args, monkeypatch):
+    def test_multiple_year_is_possible_xlsx_file(self, m_args, monkeypatch):
+        monkeypatch.setattr(
+            plt, "show", lambda: None
+        )  # prevents the test to output figure
+        ramp_main()
+
+    @mock.patch(
+        "argparse.ArgumentParser.parse_args",
+        return_value=ramp_parser.parse_args(
+            [
+                "-i",
+                os.path.join(TEST_PATH, "test_inputs", "example_ods_usecase.ods"),
+                "-y",
+                "2022",
+                "-o",
+                os.path.join(TEST_OUTPUT_PATH, "example_ods.csv"),
+            ]
+        ),
+    )
+    def test_read_year_power_input_ods_file(self, m_args, monkeypatch):
+        monkeypatch.setattr(
+            plt, "show", lambda: None
+        )  # prevents the test to output figure
+        ramp_main()
+
+    @mock.patch(
+        "argparse.ArgumentParser.parse_args",
+        return_value=ramp_parser.parse_args(
+            [
+                "-i",
+                os.path.join(TEST_PATH, "test_inputs", "example_csv_usecase.csv"),
+                "-y",
+                "2022",
+                "-o",
+                os.path.join(TEST_OUTPUT_PATH, "example_csv.csv"),
+            ]
+        ),
+    )
+    def test_read_year_power_input_csv_file(self, m_args, monkeypatch):
         monkeypatch.setattr(
             plt, "show", lambda: None
         )  # prevents the test to output figure

tests/test_inputs/example_csv_usecase.csv

@@ -0,0 +1,5 @@
+user_name,num_users,user_preference,name,number,power,num_windows,func_time,time_fraction_random_variability,func_cycle,fixed,fixed_cycle,occasional_use,flat,thermal_p_var,pref_index,wd_we_type,p_11,t_11,cw11_start,cw11_end,p_12,t_12,cw12_start,cw12_end,r_c1,p_21,t_21,cw21_start,cw21_end,p_22,t_22,cw22_start,cw22_end,r_c2,p_31,t_31,cw31_start,cw31_end,p_32,t_32,cw32_start,cw32_end,r_c3,window_1_start,window_1_end,window_2_start,window_2_end,window_3_start,window_3_end,random_var_w
+household,1,0,light,1,"[24.5,30.2,28.7,22.1,35.0,27.3,31.5,29.8,26.4,33.2,25.6,30.1,28.9,32.0,24.3,29.5,27.8,31.0,22.4,34.1,26.7,30.9,28.3,33.5,25.0,29.1,27.4,32.2,24.8,30.4,28.6,35.3,26.1,29.7,27.9,31.2,22.5,34.0,26.8,30.6,28.1,33.3,25.4,29.3,27.6,32.1,24.9,30.2,28.5,35.1,26.3,29.9,27.7,31.4,22.2,34.5,26.6,30.8,28.4,33.0,25.5,29.4,27.2,32.3,24.7,30.5,28.8,35.2,26.0,29.6,27.5,31.1,22.3,34.2,26.9,30.7,28.2,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0,35.4,26.0,24.5,30.2,28.7,22.1,35.0,27.3,31.5,29.8,26.4,33.2,25.6,30.1,28.9,32.0,24.3,29.5,27.8,31.0,22.4,34.1,26.7,30.9,28.3,33.5,25.0,29.1,27.4,32.2,24.8,30.4,28.6,35.3,26.1,29.7,27.9,31.2,22.5,34.0,26.8,30.6,28.1,33.3,25.4,29.3,27.6,32.1,24.9,30.2,28.5,35.1,26.3,29.9,27.7,31.4,22.2,34.5,26.6,30.8,28.4,33.0,25.5,29.4,27.2,32.3,24.7,30.5,28.8,35.2,26.0,29.6,27.5,31.1,22.3,34.2,26.9,30.7,28.2,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0,35.4,26.0,24.5,30.2,28.7,22.1,35.0,27.3,31.5,29.8,26.4,33.2,25.6,30.1,28.9,32.0,24.3,29.5,27.8,31.0,22.4,34.1,26.7,30.9,28.3,33.5,25.0,29.1,27.4,32.2,24.8,30.4,28.6,35.3,26.1,29.7,27.9,31.2,22.5,34.0,26.8,30.6,28.1,33.3,25.4,29.3,27.6,32.1,24.9,30.2,28.5,35.1,26.3,29.9,27.7,31.4,22.2,34.5,26.6,30.8,28.4,33.0,25.5,29.4,27.2,32.3,24.7,30.5,28.8,35.2,26.0,29.6,27.5,31.1,22.3,34.2,26.9,30.7,28.2,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0,35.4,26.0,24.5,30.2,28.7,22.1,35.0,27.3,31.5,29.8,26.4,33.2,25.6,30.1,28.9,32.0,24.3,29.5,27.8,31.0,22.4,34.1,26.7,30.9,28.3,33.5,25.0,29.1,27.4,32.2,24.8,30.4,28.6,35.3,26.1,29.7,27.9,31.2,22.5,34.0,26.8,30.6,28.1,33.3,25.4,29.3,27.6,32.1,24.9,30.2,28.5,35.1,26.3,29.9,27.7,31.4,22.2,34.5,26.6,30.8,28.4,33.0,25.5,29.4,27.2,32.3,24.7,30.5,28.8,35.2,26.0,29.6,27.5,31.1,22.3,34.2,26.9,30.7,28.2,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0,35.4,26.0,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0,35.4,26.0,33.4,25.3,29.2,27.4,32.0,24.6,30.3,28.0]",1,0,0,1,no,0,1,no,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1440,0,0,0,0,0
+household,1,0,tv,1,0,1,0,0,1,no,0,1,no,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1440,0,0,0,0,0
+school,1,0,light,1,0,1,0,0,1,no,0,1,no,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1440,0,0,0,0,0
+school,1,0,computer,1,0,1,0,0,1,no,0,1,no,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1440,0,0,0,0,0