From b836221f6d9c4919bdca3c67ee28e96dadfdecda Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Thu, 23 Feb 2023 18:40:53 -0800 Subject: [PATCH] docs update --- .zenodo.json | 5 ++ AUTHORS.rst | 5 +- CHANGELOG.rst | 6 +- README.rst | 26 ++------ citation.cff | 5 +- docs/contributing.rst | 15 +++-- docs/data.rst | 54 +++++++++-------- docs/examples.rst | 72 ++++++++++++---------- docs/functionality.rst | 131 ++++++++++++++++++++++------------------- docs/install.rst | 10 ++-- requirements.txt | 1 + setup.cfg | 3 +- src/otoole/cli.py | 2 +- 13 files changed, 176 insertions(+), 159 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index c25b3206..7361f012 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -7,6 +7,11 @@ "affiliation": "KTH Royal Institute of Technology", "orcid": "0000-0001-9367-1791" }, + { + "name": "Trevor Barnes", + "affiliation": "Simon Fraser University", + "orcid": "0000-0003-2458-2968" + }, { "name": "Hauke Henke", "affiliation": "KTH Royal Institute of Technology", diff --git a/AUTHORS.rst b/AUTHORS.rst index b438ce38..6b28858e 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,4 +2,7 @@ Contributors ============ -* Will Usher +* Will Usher +* Hauke Henke +* Christoph Muschner +* Trevor Barnes diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f367e4e6..f84de115 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,9 +4,11 @@ Changelog Version 1.0 =========== - Requires explicit provision of a user-defined configuration file for otoole to workbook -- Reads in a model configuration from a datapackage -- Fixed issue #114 so that conversion to Excel workbooks is now operating +- Deprecates datapackage fuctionality +- Adds setup command to generate template config.yaml and csv files +- Documentation update - Bumped pyscaffold to 4.2 +- Otoole now requires Python 3.8 or later Version 0.9 =========== diff --git a/README.rst b/README.rst index 8d0222e7..1756471c 100644 --- a/README.rst +++ b/README.rst @@ -24,12 +24,11 @@ Description for users of OSeMOSYS.The aim of the package is to provide commonly used pre- and post-processing steps for OSeMOSYS. -.. image:: img/osemosys_dataflow.png +.. image:: docs/_static/workflow.png **otoole** aims to support different ways of storing input data and results, -including csv files, databases, datapackages and Excel workbooks, -as well as different implementations of the OSeMOSYS model. -This improves interoperability of analyses and +including csv files, databases and Excel workbooks,as well as different implementations +of the OSeMOSYS model. This improves interoperability of analyses and generally makes life a little bit easier. Installation @@ -51,22 +50,9 @@ https://otoole.readthedocs.io/en/stable/index.html Contributing ============ -New ideas and bugs `should be submitted: `_ +New ideas and bugs `should be submitted `_ to the repository Issue Tracker. Please do contribute by discussing and developing these ideas further, or by developing the codebase. -To contribute directly to the documentation of code development, you -first need to install the package in *develop mode*:: - - git clone http://github.com/OSeMOSYS/otoole - cd otoole - git checkout - python setup.py develop - -Alternatively, use pip to install otoole from git in editable mode - - pip install -e git+http://github.com/OSeMOSYS/otoole@master#egg=otoole - -Now, all changes made in the codebase will automatically be reflected -in the installed Python version accessible on the command line or from -importing otoole modules into other Python packages. +To contribute directly to the documentation of code development, please see +the contribution guidelines document. diff --git a/citation.cff b/citation.cff index ab295245..86524daa 100644 --- a/citation.cff +++ b/citation.cff @@ -4,6 +4,9 @@ authors: - family-names: Usher given-names: Will orcid: https://orcid.org/0000-0001-9367-1791 + - family-names: Barnes + given-names: Trevor + orcid: https://orcid.org/0000-0003-2458-2968 - family-names: Henke given-names: Hauke orcid: https://orcid.org/0000-0003-0098-8701 @@ -11,6 +14,6 @@ authors: given-names: Christoph orcid: https://orcid.org/0000-0001-8144-5260 title: "otoole: OSeMOSYS tools for energy work" -version: 0.10.0 +version: 1.0.0 doi: 10.5281/zenodo.4730003 date-released: 2021-04-30 diff --git a/docs/contributing.rst b/docs/contributing.rst index 598c5046..87d2901e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -2,7 +2,7 @@ Contributing ============ -Welcome to ``otoole`` contributor's guide. +Welcome to ``otoole`` contributor's guide! This document focuses on getting any potential contributor familiarized with the development processes, but `other kinds of contributions`_ are also @@ -78,15 +78,14 @@ Code Contributions ================== ``otoole`` is built around a command line tool which is written -using the Python argparse library. The otoole.cli module is a useful +using the Python argparse library. The ``otoole.cli`` module is a useful place to start when trying to understand how each command works. The ``otoole convert`` and ``otoole results`` commands both -use classes which inherit the otoole.Strategy class. -An ``otoole.ReadStrategy`` implements functionality which returns a -dictionary of pandas.DataFrame objects. -An ``otoole.WriteStrategy`` takes a dictionary of pandas.DataFrames -as an argument and writes out the target file format. +use classes which inherit the ``otoole.Strategy`` class. +An ``otoole.ReadStrategy`` implements functionality to read in data, while an +``otoole.WriteStrategy`` writes out the target file format. The internal datastore +format in ``otool`` is a dictionary of ``pandas.DataFrames``. Comprehensive unit tests in the ``tests`` folder provide another way to understand what each of the components does. @@ -224,7 +223,7 @@ package: tox -r -e docs #. Make sure to have a reliable |tox|_ installation that uses the correct - Python version (e.g., 3.7+). When in doubt you can run:: + Python version (e.g., 3.8+). When in doubt you can run:: tox --version # OR diff --git a/docs/data.rst b/docs/data.rst index 453fe188..84ae380e 100644 --- a/docs/data.rst +++ b/docs/data.rst @@ -7,22 +7,24 @@ Data Formats User Configuration File ----------------------- +.. versionadded:: v1.0.0 + The user configuration file + Overview ~~~~~~~~ -All commands in ``otoole`` require the user to specify a configuration file that describes +Most commands in ``otoole`` require the user to specify a configuration file that describes the ``parameters``, ``sets``, and ``results`` in the model. This configuration file is -written in ``yaml`` and is typically saved as ``config.yaml``. How to define the -``parameters``, ``sets``, and ``results`` in this file are highlighted below. If the user -incorrectly enters data, validation checks in ``otoole`` should catch this. - -This section will cover how to formate the user configuration file. +written in ``yaml`` and is typically saved as ``config.yaml``. This section will cover how to +format the user configuration file. If the user incorrectly enters data, validation checks in +``otoole`` should catch this. Information Required ~~~~~~~~~~~~~~~~~~~~ -The table below highlights what information is required for each value type. -Required values are given by ``X``, optional values are given by ``(X)``. +The table below highlights what information is required for each ``Set``, +``Parameter`` and ``Result`` definition in the configuration file. Required values are +given by **X**, while optional values are given by **(X)**. +-------------+------+------------+---------+ | | Set | Parameter | Result | @@ -41,12 +43,12 @@ Required values are given by ``X``, optional values are given by ``(X)``. +-------------+------+------------+---------+ .. WARNING:: - Names that are longer than 31 characters require a ``short_name`` field. This is due + Names longer than 31 characters require a ``short_name`` field. This is due to character limits on excel sheet names. ``otoole`` will raise an error if a ``short_name`` is not provided in these instances. -Sets -~~~~ +Sets Format +~~~~~~~~~~~ Sets are defined as follows:: @@ -56,12 +58,12 @@ Sets are defined as follows:: type: set .. NOTE:: - It's convention in OSeMOSYS to give sets uppercase names + It's convention in OSeMOSYS to capitalize set names -Parmaters -~~~~~~~~~ +Parmaters Foramt +~~~~~~~~~~~~~~~~ -Parameters are defined as follows. When referencing set indices, use the full +Parameters are defined as follows. When referencing set indices use the full name, **not** the ``short_name``:: ParameterName: @@ -72,12 +74,12 @@ name, **not** the ``short_name``:: default: 0 .. NOTE:: - It's convention in OSeMOSYS to use Pascal case for parameters + It's convention in OSeMOSYS to use Pascal case for parameter names -Results -~~~~~~~ +Results Format +~~~~~~~~~~~~~~ -Results are defined as follows. When referencing set indices, use the full +Results are defined as follows. When referencing set indices use the full name, **not** the ``short_name``:: AnnualEmissions: @@ -89,13 +91,12 @@ name, **not** the ``short_name``:: calculated: "True" or "False" .. NOTE:: - It's convention in OSeMOSYS to use Pascal case for results - + It's convention in OSeMOSYS to use Pascal case for result names Examples ~~~~~~~~ -Below are examples of correctly formatted configuration file values. See the Simplicity +Below are examples of correctly formatted configuration file values. See the Simplicity_ repository for a complete example. 1. Set definition of ``TECHNOLOGY``:: @@ -131,8 +132,11 @@ Overview This section will describe how to format data for ``excel``, ``csv``, and ``datafile`` formats. -.. NOTE:: - See the Simplicity_ repository for a full example of all these formats. +.. deprecated:: v1.0.0 + The ``datapackage`` format is no longer supported + +.. SEEALSO:: + See the Simplicity_ repository for a full example of these formats Excel ~~~~~ @@ -325,7 +329,7 @@ And in the same ``data.txt`` file, the set ``TECHNOLOGY`` will be defined as fol HYD2 ... -.. TIP:: +.. SEEALSO:: For reading and writing between Python and AMPL_, see the amply_ Python package. .. _MathProg: https://en.wikibooks.org/wiki/GLPK/GMPL_(MathProg) diff --git a/docs/examples.rst b/docs/examples.rst index f16ad295..b1bf90c9 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -4,9 +4,18 @@ Examples -------- -This page will present examples to show the full functionality of ``otoole``. -To follow these examples, clone the Simplicity_ repository and run all commands -from the ``simplicity/`` directory. +This page will present examples to show the full functionality of ``otoole``. It will +walk through the ``convert``, ``results``, ``setup``, ``viz`` and ``validate`` +functionality in seperate simple use cases. + +.. NOTE:: + To follow these examples, clone the Simplicity_ repository and run all commands + from the ``simplicity/`` directory. + +.. CAUTION:: + While ``otoole`` does not require any solver, these examples will use the + free and open-source tools GLPK_ and CBC_ to build and solve example + models. Data Conversion --------------- @@ -55,7 +64,7 @@ Result Processing Objective ~~~~~~~~~ -Use an excel worksheet to build and solve an OSeMOSYS model with CPLEX. Only +Use an excel worksheet to build and solve an OSeMOSYS model with CBC. Only generate the results for ``TotalDiscountedCost`` and ``TotalCapacityAnnual``. 1. ``otoole`` Convert @@ -73,9 +82,9 @@ Use GLPK_ to build the model and save it as ``simplicity.lp``:: 3. Solve the Model ~~~~~~~~~~~~~~~~~~ -Use CPLEX_ to solve the model and save the solution file as ``simplicity.sol``:: +Use CBC_ to solve the model and save the solution file as ``simplicity.sol``:: - $ cplex -c "read simplicity.sol" "optimize" "write simplicity.sol" + $ cbc simplicity.sol solve -solu simplicity.sol 4. Modify the Configuration File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -95,16 +104,12 @@ Create a folder to hold the results:: $ mkdir results -Sort the solution file (only needed for CPLEX solution files):: - - $ simplicity.sol sort > simplicity_sorted.sol - Use ``otoole``'s ``result`` package to generate the result CSVs:: - $ otoole results cplex csv simplicity_sorted.sol results config.yaml + $ otoole results cbc csv simplicity.sol results config.yaml -Input Data Template Setup -------------------------- +Template Setup +-------------- Objective ~~~~~~~~~ @@ -114,7 +119,7 @@ Generate a template configuration file and excel input file to use with 1. Create the Configuration File ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Run the following command, wto create a template configuration file +Run the following command, to create a template configuration file called ``config.yaml``:: $ otoole setup config config.yaml @@ -123,7 +128,7 @@ called ``config.yaml``:: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``otoole`` will only generate template CSV data, however, we want to input data in Excel format. Therefore, we will first generate CSV data and convert -it to Excel format.:: +it to Excel format:: $ otoole setup csv data @@ -133,25 +138,24 @@ Open up the the file ``data/YEARS.csv`` and add all the years over the model horizon. For example, if the model horizon is from 2020 to 2050, the ``data/YEARS.csv`` file should be formatted as follows: -+-------+ -| VALUE | -+=======+ -| 2020 | -|-------| -| 2021 | -|-------| -| 2022 | -|-------| -| ... | -|-------| -| 2050 | -+-------+ ++---------+ +| VALUE | ++=========+ +| 2020 | ++---------+ +| 2021 | ++---------+ +| 2022 | ++---------+ +| ... | ++---------+ +| 2050 | ++---------+ .. NOTE:: While this step in not technically required, by filling out the years in CSV format, ``otoole`` will pivot all the Excel sheets on the years - you input during the conversion process. This will save significant - formatting time! + during the conversion process. This will save significant formatting time! 4. Convert the CSV Template Data ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -193,11 +197,15 @@ displayed Model Validation ---------------- +.. NOTE:: + In this example, we will use a very simple model instead of the + Simplicity_ demonstration model. This way the user does not need to be + familar with the naming convenations of the model. + Objective ~~~~~~~~~ -Use ``otoole`` to validate an input data file. In this example, we will use a -very simple model instead of the Simplicity_ demonstration model. The model +Use ``otoole`` to validate an input data file. The model we are going to validate is shown below, where the fuel and technology codes are shown in bold face. diff --git a/docs/functionality.rst b/docs/functionality.rst index 71f9f1e9..157c252c 100644 --- a/docs/functionality.rst +++ b/docs/functionality.rst @@ -5,10 +5,8 @@ Core Functionality ================== ``otoole``'s functionality includes converting between data formats, processing -solution files, and visualising the model. These core functions are described on the page. -``otoole`` is a command-line tool. By writing commands in a terminal window, -you can tell ``otoole`` to perform certain tedious tasks for you to turbo-charge -your OSeMOSYS modelling. +solution files, and visualising the model. These core functions, which provide a method +to turbo-charge your modelling, are described on this page! As shown in the diagram, ``otoole`` deals primarily with data before and after OSeMOSYS. Data work prior to the generation of a datafile, which is read in by GLPK_, is called @@ -18,19 +16,12 @@ Gurobi_, is called results and post-processing. .. image:: _static/workflow.png -Configuration File ------------------- - -When performing any operation with ``otoole``, the user must supply a configuration -file. This file specifies what ``sets``, ``parameters``, and ``results`` are in the -model. The configuration file allows modellers to easily switch between -OSeMOSYS versions that may have different structures. - -.. TIP:: - The :ref:`dataformats` page provides information on how to format the configuration file +.. NOTE:: + While ``otoole`` is targetted at OSeMOSYS users, the functionality can eaisly be extended + to work with any workflow that involves the use of a MathProg file! -Pre-processing --------------- +Data Conversion +--------------- Overview ~~~~~~~~ @@ -44,6 +35,9 @@ conversion between the following formats: - A folder of CSV files - GNU MathProg datafile +.. deprecated:: v1.0.0 + The ``datapackage`` format is no longer supported + ``otoole convert`` ~~~~~~~~~~~~~~~~~~ @@ -63,16 +57,13 @@ input formats:: optional arguments: -h, --help show this help message and exit --write_defaults Writes default values + --keep_whitespace Keeps leading/trailing whitespace -.. NOTE:: - The ``config`` positional argument is required from version 1.0 and onwards - -.. TIP:: - The :ref:`dataformats` page provides information on how to structure input data +.. versionadded:: v1.0.0 + The ``config`` positional argument is now required - -Post-processing ---------------- +Result Processing +----------------- Overview ~~~~~~~~ @@ -98,11 +89,11 @@ so as to speed up the model matrix generation and solution times. ``otoole results`` ~~~~~~~~~~~~~~~~~~ -The ``results`` command creates a folder of CSV result files from a CBC, CLP, Gurobi or CPLEX -solution file:: +The ``results`` command creates a folder of CSV result files from a CBC_, CLP_, +Gurobi_ or CPLEX_ solution file:: $ otoole results --help - usage: otoole results [-h] [--input_datafile INPUT_DATAFILE] [--write_defaults] {cbc,cplex,gurobi} {csv} from_path to_path config + usage: otoole results [-h] [--input_datafile INPUT_DATAFILE] [--input_datapackage INPUT_DATAPACKAGE] [--write_defaults] {cbc,cplex,gurobi} {csv} from_path to_path config positional arguments: {cbc,cplex,gurobi} Result data format to convert from @@ -112,14 +103,52 @@ solution file:: config Path to config YAML file optional arguments: - -h, --help show this help message and exit - --input_datafile INPUT_DATAFILE - Input GNUMathProg datafile required for OSeMOSYS short or fast results - --write_defaults Writes default values + -h, --help show this help message and exit + --input_datafile INPUT_DATAFILE + Input GNUMathProg datafile required for OSeMOSYS short or fast results + --input_datapackage INPUT_DATAPACKAGE + Deprecated + --write_defaults Writes default values + +.. versionadded:: v1.0.0 + The ``config`` positional argument is now required + +.. deprecated:: v1.0.0 + The ``--input_datapackage`` flag is no longer supported .. WARNING:: - If using CPLEX_, note that you need to first sort the CPLEX file which you can do from - the command line. See the :ref:`examples` page for a full CPLEX_ workflow example. + If using CPLEX_, you will need to transform and sort the solution file before + processing it with ``otoole``. Instructions on how to run the transformation + script are on the `OSeMOSYS Repository`_. After transformation, sort the file + with the command ``sort > ``. + +Setup +----- +The ``setup`` module in ``otoole`` allows you to generate template files to +quickly get up and running. + +``otoole setup`` +~~~~~~~~~~~~~~~~ +The ``setup``command allows you to generate a template user configuration file, +useful for ``conversion`` and ``result`` commands, and template input ``csv`` +data:: + + $ otoole setup --help + + usage: otoole setup [-h] [--write_defaults] [--overwrite] {config,csv} data_path + + positional arguments: + {config,csv} Type of file to setup + data_path Path to file or folder to save to + + optional arguments: + -h, --help show this help message and exit + --write_defaults Writes default values + --overwrite Overwrites existing data + +.. WARNING:: + The template files are generated based on a specific version of OSeMOSYS, users will + need to adapt the template data for their own needs Visualization ------------- @@ -150,13 +179,13 @@ visualising the reference energy system through the ``vis res`` command is suppo .. NOTE:: The ``resfile`` command should include a file ending used for images, - including ``bmp``, ``jpg``, ``pdf``, ``png`` etc. The ``graphviz`` library + including ``bmp``, ``jpg``, ``pdf``, ``png`` etc. The graphviz_ library used to layout the reference energy system will interpret the file ending. Validation ---------- -The validation module in ``otoole`` checks the technology and fuel names -against a standard or user defined configuration file. +The validation module in ``otoole`` checks technology and fuel names against a +standard or user defined configuration file. ``otoole validate`` ~~~~~~~~~~~~~~~~~~~ @@ -179,33 +208,11 @@ the rest of the model:: --validate_config VALIDATE_CONFIG Path to a user-defined validation-config file -Setup ------ -The ``setup`` module in ``otoole`` allows you to generate template files to -quickly get up and running with ``otoole``. - -``otoole setup`` -~~~~~~~~~~~~~~~~ -The ``setup``command allows you to generate a template user configuration file, -useful for ``conversion`` and ``result`` commands, and template input ``csv`` -data. **Note:** the data generated based on a specific version of OSeMOSYS, -users will need to adapt the template data for their own needs:: - - $ otoole setup --help - - usage: otoole setup [-h] [--write_defaults] [--overwrite] {config,csv} data_path - - positional arguments: - {config,csv} Type of file to setup - data_path Path to file or folder to save to - - optional arguments: - -h, --help show this help message and exit - --write_defaults Writes default values - --overwrite Overwrites existing data - .. _GLPK: https://www.gnu.org/software/glpk/ .. _CBC: https://github.com/coin-or/Cbc +.. _CLP: https://github.com/coin-or/Clp .. _CPLEX: https://www.ibm.com/products/ilog-cplex-optimization-studio/cplex-optimizer -.. _Gurobi: https://www.gurobi.com/" +.. _Gurobi: https://www.gurobi.com/ +.. _`OSeMOSYS Repository`: https://github.com/OSeMOSYS/OSeMOSYS_GNU_MathProg/tree/master/scripts +.. _graphviz: https://graphviz.org/ diff --git a/docs/install.rst b/docs/install.rst index 33dadc5a..81a256c8 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -12,16 +12,14 @@ Install ``otoole`` using pip:: Check the version installed:: - otoole -V - - ~/repositories/otoole$ otoole -V - 0.11.0 + ~ otoole -V + 1.0.0 To upgrade ``otoole`` using pip:: pip install otoole --upgrade -.. NOTE:: +.. TIP:: We recommend installing ``otoole`` in an isolated virtual environment, either through the use of venv_ or conda_ @@ -34,7 +32,7 @@ individual dependencies, the easiest way to do this is through miniconda_. 1. Obtain the miniconda_ package 2. Add the ``conda-forge`` channel ``conda config --add channels conda-forge`` 3. Create a new Python environment - ``conda create -n otoole python=3.7 networkx pandas graphviz=2.46.1 xlrd pydantic`` + ``conda create -n otoole python>3.7 networkx pandas graphviz=2.46.1 xlrd pydantic`` 4. Activate the new environment ``conda activate otoole`` 5. Use pip to install otoole ``pip install otoole`` diff --git a/requirements.txt b/requirements.txt index 92b51573..9005e104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,6 @@ pandas pandas_datapackage_reader pydantic pydot +python>=3.8 pyyaml xlrd diff --git a/setup.cfg b/setup.cfg index 1dff345a..02415aab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ package_dir = =src # Require a min/specific Python version (comma-separated conditions) -python_requires = >=3.7 +python_requires = >=3.8 # Add here dependencies of your project (line-separated), e.g. requests>=2.2,<3.0. # Version specifiers like >=2.2,<3.0 avoid problems due to API changes in @@ -52,6 +52,7 @@ install_requires = flatten_dict openpyxl pydantic + python>=3.8 [options.packages.find] where = src exclude = diff --git a/src/otoole/cli.py b/src/otoole/cli.py index c422820b..8d0ce16c 100644 --- a/src/otoole/cli.py +++ b/src/otoole/cli.py @@ -378,7 +378,7 @@ def get_parser(): ) convert_parser.add_argument( "--keep_whitespace", - help="Keeps leading/trailing whitespace in CSV files", + help="Keeps leading/trailing whitespace", default=False, action="store_true", )