From 01960d041464b458f267ef8c0bbd94213bff8616 Mon Sep 17 00:00:00 2001 From: Peter Mitri Date: Mon, 2 Sep 2024 16:12:56 +0200 Subject: [PATCH] Add cucumber tests tool (#2347) --- .github/workflows/cucumber-tests/action.yml | 23 ++++++ .github/workflows/ubuntu.yml | 16 ++++- .github/workflows/windows-vcpkg.yml | 16 ++++- .gitmodules | 3 + src/tests/CMakeLists.txt | 3 +- src/tests/cucumber/CMakeLists.txt | 2 + .../cucumber/features/medium_tests.feature | 15 ++++ .../cucumber/features/short_tests.feature | 45 ++++++++++++ .../cucumber/features/steps/assertions.py | 4 ++ .../cucumber/features/steps/context_utils.py | 22 ++++++ .../cucumber/features/steps/output_utils.py | 37 ++++++++++ .../features/steps/simulator_utils.py | 47 +++++++++++++ src/tests/cucumber/features/steps/steps.py | 67 ++++++++++++++++++ .../features/steps/study_input_handler.py | 31 ++++++++ src/tests/cucumber/readme.md | 70 +++++++++++++++++++ src/tests/cucumber/requirements.txt | 2 + .../resources/Antares_Simulator_Tests_NR | 1 + src/tests/run-study-tests/readme.md | 2 +- 18 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cucumber-tests/action.yml create mode 100644 src/tests/cucumber/CMakeLists.txt create mode 100644 src/tests/cucumber/features/medium_tests.feature create mode 100644 src/tests/cucumber/features/short_tests.feature create mode 100644 src/tests/cucumber/features/steps/assertions.py create mode 100644 src/tests/cucumber/features/steps/context_utils.py create mode 100644 src/tests/cucumber/features/steps/output_utils.py create mode 100644 src/tests/cucumber/features/steps/simulator_utils.py create mode 100644 src/tests/cucumber/features/steps/steps.py create mode 100644 src/tests/cucumber/features/steps/study_input_handler.py create mode 100644 src/tests/cucumber/readme.md create mode 100644 src/tests/cucumber/requirements.txt create mode 160000 src/tests/resources/Antares_Simulator_Tests_NR diff --git a/.github/workflows/cucumber-tests/action.yml b/.github/workflows/cucumber-tests/action.yml new file mode 100644 index 0000000000..475d3310b3 --- /dev/null +++ b/.github/workflows/cucumber-tests/action.yml @@ -0,0 +1,23 @@ +name: "Run cucumber tests" +description: "Run cucumber tests" +inputs: + feature: + description: 'Feature file or folder to run (default runs all features in "features" folder)' + required: false + default: 'features' + tags: + description: 'Tags to run (default skips tests marked @flaky)' + required: false + default: '~@flaky' +runs: + using: "composite" + steps: + - name: Install Python requirements + shell: bash + run: python3 -m pip install -r src/tests/cucumber/requirements.txt + + - name: Run tests + shell: bash + run: | + cd src/tests/cucumber + behave --tags ${{ inputs.tags }} ${{ inputs.feature }} diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 01d2b3a10e..2c73097adb 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -142,10 +142,14 @@ jobs: run: | echo "SIMTEST=${{ fromJson(env.SIMTEST_JSON).version }}" >> $GITHUB_ENV - - name: Init submodule + - name: Init submodule Antares_Simulator_Tests run: | git submodule update --init --remote --recursive src/tests/resources/Antares_Simulator_Tests + - name: Init submodule Antares_Simulator_Tests_NR + run: | + git submodule update --init --remote --recursive src/tests/resources/Antares_Simulator_Tests_NR + - name: Run named mps tests if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }} uses: ./.github/workflows/run-tests @@ -231,6 +235,11 @@ jobs: batch-name: short-tests os: ${{ env.os }} + - name: Run cucumber on short-tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/short_tests.feature" + - name: Run mps tests if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }} uses: ./.github/workflows/run-tests @@ -273,6 +282,11 @@ jobs: batch-name: medium-tests os: ${{ env.os }} + - name: Run cucumber on medium-tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/medium_tests.feature" + - name: Run long-tests-1 if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }} uses: ./.github/workflows/run-tests diff --git a/.github/workflows/windows-vcpkg.yml b/.github/workflows/windows-vcpkg.yml index eadd043a69..e5cf8fea3f 100644 --- a/.github/workflows/windows-vcpkg.yml +++ b/.github/workflows/windows-vcpkg.yml @@ -110,10 +110,14 @@ jobs: - name: Install pip dependencies if necessary run: pip install -r src/tests/examples/requirements.txt - - name: Init submodule + - name: Init submodule Antares_Simulator_Tests run: | git submodule update --init --remote src/tests/resources/Antares_Simulator_Tests + - name: Init submodule Antares_Simulator_Tests_NR + run: | + git submodule update --init --remote src/tests/resources/Antares_Simulator_Tests_NR + - name: Enable git longpaths run: git config --system core.longpaths true @@ -241,6 +245,11 @@ jobs: batch-name: short-tests os: ${{ env.os }} + - name: Run cucumber on short-tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/short_tests.feature" + - name: Run mps tests if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }} uses: ./.github/workflows/run-tests @@ -275,6 +284,11 @@ jobs: batch-name: medium-tests os: ${{ env.os }} + - name: Run cucumber on medium-tests + uses: ./.github/workflows/cucumber-tests + with: + feature: "features/medium_tests.feature" + - name: Run long-tests-1 if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }} uses: ./.github/workflows/run-tests diff --git a/.gitmodules b/.gitmodules index 77fb4921c7..16639d0fb1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "vcpkg"] path = vcpkg url = https://github.com/microsoft/vcpkg.git +[submodule "src/tests/resources/Antares_Simulator_Tests_NR"] + path = src/tests/resources/Antares_Simulator_Tests_NR + url = https://github.com/AntaresSimulatorTeam/Antares_Simulator_Tests_NR.git diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index fcbc5cd6b2..2babe91fab 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -9,8 +9,7 @@ if (Boost_FOUND) set_target_properties(Boost::unit_test_framework PROPERTIES IMPORTED_GLOBAL TRUE) endif() - - +add_subdirectory(cucumber) # end to end test require boost 1.6.x because of boost::test_tools::tolerance, BOOST_TEST # old versions of Boost don't contain a '.', also ignore them diff --git a/src/tests/cucumber/CMakeLists.txt b/src/tests/cucumber/CMakeLists.txt new file mode 100644 index 0000000000..cf22c8415b --- /dev/null +++ b/src/tests/cucumber/CMakeLists.txt @@ -0,0 +1,2 @@ +file(GENERATE OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/conf.yaml CONTENT "antares-solver : $" + CONDITION $,${CMAKE_BUILD_TYPE}>) \ No newline at end of file diff --git a/src/tests/cucumber/features/medium_tests.feature b/src/tests/cucumber/features/medium_tests.feature new file mode 100644 index 0000000000..98e8a6fb26 --- /dev/null +++ b/src/tests/cucumber/features/medium_tests.feature @@ -0,0 +1,15 @@ +Feature: medium tests + + @fast @medium @incomplete + Scenario: 035 Mixed Expansion - Smart grid model 2 + Given the study path is "medium-tests/035 Mixed Expansion - Smart grid model 2" + When I run antares simulator + Then the simulation takes less than 15 seconds + And the simulation succeeds + And the expected value of the annual system cost is 3.725e+10 + And the minimum annual system cost is 3.642e+10 + And the maximum annual system cost is 4.011e+10 + And the annual system cost is + | EXP | STD | MIN | MAX | + | 3.725e+10 | 1.063e+09 | 3.642e+10 | 4.011e+10 | + # TODO : add steps when we understand what this test is supposed to test \ No newline at end of file diff --git a/src/tests/cucumber/features/short_tests.feature b/src/tests/cucumber/features/short_tests.feature new file mode 100644 index 0000000000..8a1d65d6c7 --- /dev/null +++ b/src/tests/cucumber/features/short_tests.feature @@ -0,0 +1,45 @@ +Feature: short tests + + @fast @short + Scenario: 001 One node - passive + Given the study path is "short-tests/001 One node - passive" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is + | EXP | STD | MIN | MAX | + | 0 | 0 | 0 | 0 | + + @fast @short + Scenario: 002 Thermal fleet - Base + Given the study path is "short-tests/002 Thermal fleet - Base" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is + | EXP | STD | MIN | MAX | + | 2.729e+7 | 0 | 2.729e+7 | 2.729e+7 | + And in area "AREA", during year 1, loss of load lasts 1 hours + And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW + + @fast @short + Scenario: 003 Thermal fleet - Must-run + Given the study path is "short-tests/003 Thermal fleet - Must-run" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is + | EXP | STD | MIN | MAX | + | 2.751e+7 | 0 | 2.751e+7 | 2.751e+7 | + And in area "AREA", during year 1, loss of load lasts 1 hours + And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW + + @fast @short + Scenario: 021 Four areas - DC law + Given the study path is "short-tests/021 Four areas - DC law" + When I run antares simulator + Then the simulation takes less than 20 seconds + And the simulation succeeds + And the annual system cost is + | EXP | STD | MIN | MAX | + | 7.972e+10 | 2.258e+10 | 5.613e+10 | 1.082e+11 | \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/assertions.py b/src/tests/cucumber/features/steps/assertions.py new file mode 100644 index 0000000000..dda7d0232c --- /dev/null +++ b/src/tests/cucumber/features/steps/assertions.py @@ -0,0 +1,4 @@ +# Custom assertions + +def assert_double_close(expected, actual, relative_tolerance): + assert abs((actual - expected) / max(1e-6, expected)) <= relative_tolerance \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/context_utils.py b/src/tests/cucumber/features/steps/context_utils.py new file mode 100644 index 0000000000..5a7af9dfe3 --- /dev/null +++ b/src/tests/cucumber/features/steps/context_utils.py @@ -0,0 +1,22 @@ +# Manage cached output data in "context" object + +from output_utils import * + +def get_annual_system_cost(context): + if context.annual_system_cost is None: + context.annual_system_cost = parse_annual_system_cost(context.output_path) + return context.annual_system_cost + +def get_hourly_values_for_specific_hour(context, area : str, year : int, date : str): + df = get_hourly_values(context, area, year) + day, month, hour = date.split(" ") + return df.loc[(df['Unnamed: 2'] == int(day)) & (df['Unnamed: 3'] == month) & (df['Unnamed: 4'] == hour)] + +def get_hourly_values(context, area : str, year : int): + if context.hourly_values is None: + context.hourly_values = {} + if area not in context.hourly_values: + context.hourly_values[area] = {} + if year not in context.hourly_values[area]: + context.hourly_values[area][year] = parse_hourly_values(context.output_path, area, year) + return context.hourly_values[area][year] \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/output_utils.py b/src/tests/cucumber/features/steps/output_utils.py new file mode 100644 index 0000000000..778a1d36b9 --- /dev/null +++ b/src/tests/cucumber/features/steps/output_utils.py @@ -0,0 +1,37 @@ +# Antares outputs parsing + +import os +import pandas +import configparser + +def parse_output_folder_from_logs(logs: bytes) -> str: + for line in logs.splitlines(): + if b'Output folder : ' in line: + return line.split(b'Output folder : ')[1].decode('ascii') + raise LookupError("Could not parse output folder in output logs") + + +def parse_annual_system_cost(output_path: str) -> dict: + file = open(os.path.join(output_path, "annualSystemCost.txt"), 'r') + keys = ["EXP", "STD", "MIN", "MAX"] + annual_system_cost = {} + for line in file.readlines(): + for key in keys: + if key in line: + annual_system_cost[key] = float(line.split(key + " : ")[1]) + return annual_system_cost + + +def parse_simu_time(output_path: str) -> float: + execution_info = configparser.ConfigParser() + execution_info.read(os.path.join(output_path, "execution_info.ini")) + return float(execution_info['durations_ms']['total']) / 1000 + + +def parse_hourly_values(output_path: str, area: str, year: int): + return read_csv(os.path.join(output_path, "economy", "mc-ind", f"{year:05d}", "areas", area, "values-hourly.txt")) + + +def read_csv(file_name): + ignore_rows = [0, 1, 2, 3, 5, 6] + return pandas.read_csv(file_name, skiprows=ignore_rows, sep='\t', low_memory=False) \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/simulator_utils.py b/src/tests/cucumber/features/steps/simulator_utils.py new file mode 100644 index 0000000000..91da69a5ad --- /dev/null +++ b/src/tests/cucumber/features/steps/simulator_utils.py @@ -0,0 +1,47 @@ +# Methods to run Antares simulator + +import subprocess +import glob +import yaml +from pathlib import Path +from study_input_handler import study_input_handler +from output_utils import parse_output_folder_from_logs + + +def get_solver_path(): + with open("conf.yaml") as file: + content = yaml.full_load(file) + return content.get("antares-solver") + +SOLVER_PATH = get_solver_path() # we only need to run this once + + +def run_simulation(context): + activate_simu_outputs(context) # TODO : remove this and update studies instead + command = build_antares_solver_command(context) + print(f"Running command: {command}") + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + out, err = process.communicate() + context.output_path = parse_output_folder_from_logs(out) + context.return_code = process.returncode + context.annual_system_cost = None + context.hourly_values = None + + +def activate_simu_outputs(context): + sih = study_input_handler(Path(context.study_path)) + sih.set_value(variable="synthesis", value="true", file_nick_name="general") + sih.set_value(variable="year-by-year", value="true", file_nick_name="general") + + +def build_antares_solver_command(context): + command = [SOLVER_PATH, "-i", str(context.study_path)] + if context.use_ortools: + command.append('--use-ortools') + command.append('--ortools-solver=' + context.ortools_solver) + if context.named_mps_problems: + command.append('--named-mps-problems') + if context.parallel: + command.append('--force-parallel=4') + return command + diff --git a/src/tests/cucumber/features/steps/steps.py b/src/tests/cucumber/features/steps/steps.py new file mode 100644 index 0000000000..f49bed8868 --- /dev/null +++ b/src/tests/cucumber/features/steps/steps.py @@ -0,0 +1,67 @@ +# Gherkins test steps definitions + +import os +from behave import * +from simulator_utils import * +from assertions import * +from context_utils import * + +@given('the study path is "{string}"') +def study_path_is(context, string): + context.study_path = os.path.join("..", "resources", "Antares_Simulator_Tests_NR" , string.replace("/", os.sep)) + +@when('I run antares simulator') +def run_antares(context): + context.use_ortools = True + context.ortools_solver = "sirius" + context.named_mps_problems = False + context.parallel = False + run_simulation(context) + +@then('the simulation succeeds') +def simu_success(context): + assert context.return_code == 0 + +@then('the simulation fails') +def simu_success(context): + assert context.return_code != 0 + +@then('the expected value of the annual system cost is {value}') +def check_annual_cost_expected(context, value): + assert_double_close(float(value), get_annual_system_cost(context)["EXP"], 0.001) + +@then('the minimum annual system cost is {value}') +def check_annual_cost_min(context, value): + assert_double_close(float(value), get_annual_system_cost(context)["MIN"], 0.001) + +@then('the maximum annual system cost is {value}') +def check_annual_cost_max(context, value): + assert_double_close(float(value), get_annual_system_cost(context)["MAX"], 0.001) + +@then('the annual system cost is') +def check_annual_cost(context): + for row in context.table: + assert_double_close(float(row["EXP"]), get_annual_system_cost(context)["EXP"], 0.001) + assert_double_close(float(row["STD"]), get_annual_system_cost(context)["STD"], 0.001) + assert_double_close(float(row["MIN"]), get_annual_system_cost(context)["MIN"], 0.001) + assert_double_close(float(row["MAX"]), get_annual_system_cost(context)["MAX"], 0.001) + +@then('the simulation takes less than {seconds} seconds') +def check_simu_time(context, seconds): + actual_simu_time = parse_simu_time(context.output_path) + assert actual_simu_time <= float(seconds) + +@then('in area "{area}", during year {year}, loss of load lasts {lold_hours} hours') +def check_lold_duration(context, area, year, lold_hours): + assert int(lold_hours) == get_hourly_values(context, area.lower(), int(year))["LOLD"].sum() + +@then('in area "{area}", unsupplied energy on "{date}" of year {year} is of {lold_value_mw} MW') +def check_lold_value(context, area, date, year, lold_value_mw): + actual_unsp_energ = get_hourly_values_for_specific_hour(context, area.lower(), int(year), date)["UNSP. ENRG"].sum() + assert_double_close(float(lold_value_mw), actual_unsp_energ, 0.001) + +def after_feature(context, feature): + # post-processing a test: clean up output files to avoid taking up all the disk space + if (context.output_path != None): + pathlib.Path.rmdir(context.output_path) + diff --git a/src/tests/cucumber/features/steps/study_input_handler.py b/src/tests/cucumber/features/steps/study_input_handler.py new file mode 100644 index 0000000000..90f27d7221 --- /dev/null +++ b/src/tests/cucumber/features/steps/study_input_handler.py @@ -0,0 +1,31 @@ +# Currently used to activate simulation outputs +# TODO : remove this and update parameters in simulation input files + +import os + + +class study_input_handler: + def __init__(self, study_root_directory): + self.study_root_dir = study_root_directory + self.name = os.path.basename(study_root_directory) + self.files_path = {} + self.files_path["desktop"] = self.study_root_dir / "Desktop.ini" + self.files_path["general"] = self.study_root_dir / "settings" / "generaldata.ini" + self.files_path["study"] = self.study_root_dir / "study.antares" + + def set_value(self, variable, value, file_nick_name): + # File path + file = self.files_path[file_nick_name] + # Content to print in file (tmp content) + content_out = [] + # Reading the file content (content in) + with open(file) as f: + # Searching variable and setting its value in a tmp content + for line in f: + if line.strip().startswith(variable): + content_out.append(variable + " = " + value + "\n") + else: + content_out.append(line) + # Erasing file content with the tmp content (content out) + with open(file, "w") as f: + f.writelines(content_out) diff --git a/src/tests/cucumber/readme.md b/src/tests/cucumber/readme.md new file mode 100644 index 0000000000..7ed6304d9f --- /dev/null +++ b/src/tests/cucumber/readme.md @@ -0,0 +1,70 @@ +# Antares Cucumber Tests + +This module contains non-regression functional tests for Antares written in the [Gherkin language](https://cucumber.io/docs/gherkin/). + +## Tests structure + +### Features, scenarios and tags +Features are supposed to represent big-picture features of the application. Every feature can have its own set of tests, +defined in a `.feature` file. Features are under the [features](./features) folder. +Every feature has multiple scenarios (every scenario represents a test case). +A scenario can be tagged in order to add it to a category, allowing us to run the tests on a filtered subset of the +scenarios later. The tags currently used in Antares are: +- @fast: tests that run fast +- @slow: tests that run slow +- @short: tests from the legacy "short-tests" batch +- @medium: tests from the legacy "medium-tests" batch +- @flaky: quarantine for flaky tests (i.e. sometimes pass, sometimes fail) that are to be skipped by the CI + +### Steps structure +Currently, tests are being migrated from the [legacy non-regression testing process](../run-study-tests). Thus, they +all begin by defining the path to the study to run and then call antares, through the following "steps": +~~~gherkin +Given the study path is "someFolder/someStudy" +When I run antares simulator +~~~ +The test will load the study, run antares-simulator, and hold on to its outputs. +Next, assertion "steps" can be added. For example, this next assertion checks that the simulation time (as measured by +antares-simulator and reported in its logs) is less than 15 seconds: +~~~gherkin +Then the simulation takes less than 15 seconds +~~~ +And the next step checks the expected value of the annual system cost: +~~~gherkin +Then the expected value of the annual system cost is 3.725e+10 +~~~ + +## Running the tests +### On your PC +First, you need to build antares-simulator. The tests will run the last antares-simulator executable built by the Cmake +projet. +Then, if needed, install the requirements by running: +~~~bash +pip install -r requirements.txt +~~~ +Then just run the following to execute the tests: +~~~bash +cd src/tests/cucumber +behave +~~~ +If you want to filter on a feature file and given tags, you can use: +~~~bash +behave --tags @some-tag features/some_feature.feature +~~~ +Refer to the [behave documentation](https://behave.readthedocs.io/en/latest/) for more information. + +### In the CI +Cucumber tests are run in the same way as the legacy tests in the Ubuntu & Windows CIs, except that they don't need the +reference values from the SimTest repository, since reference values are stored explicitly in the feature files. +Note that tests marked as "@flaky" are skipped by default. +Workflow file: [here](../../../.github/workflows/cucumber-tests/action.yml) + +## Under the hood +### Test files +Tests are hosted in the [Antares_Simulator_Tests_NR submodule](https://github.com/AntaresSimulatorTeam/Antares_Simulator_Tests_NR) +into the `src/test/resources` folder. Adding or modifying a study should thus change contents of this submodule. + +### Code-behind +All Gherkin steps have a code-behind definition called "step definitions". These are defined in the python files under +[features/steps](./features/steps) and use the [behave](https://behave.readthedocs.io/en/latest/) implementation of +cucumber. Feel free to add extra steps for your tests. diff --git a/src/tests/cucumber/requirements.txt b/src/tests/cucumber/requirements.txt new file mode 100644 index 0000000000..fd5e363e68 --- /dev/null +++ b/src/tests/cucumber/requirements.txt @@ -0,0 +1,2 @@ +behave +pyyaml \ No newline at end of file diff --git a/src/tests/resources/Antares_Simulator_Tests_NR b/src/tests/resources/Antares_Simulator_Tests_NR new file mode 160000 index 0000000000..9983782bc0 --- /dev/null +++ b/src/tests/resources/Antares_Simulator_Tests_NR @@ -0,0 +1 @@ +Subproject commit 9983782bc07c62f99854b1c7394d320830a40e14 diff --git a/src/tests/run-study-tests/readme.md b/src/tests/run-study-tests/readme.md index 5701d722d7..d2536dab64 100644 --- a/src/tests/run-study-tests/readme.md +++ b/src/tests/run-study-tests/readme.md @@ -212,7 +212,7 @@ The schema can be found at : **src/tests/run-study-tests/parse_studies/json_sche ```bash > cd src/tests/run-study-tests -> python -m pytest -m mps --solver-path=/path/to/the/Antares/solver/antares-x.y-solver.exe +> python -m pytest -m json --solver-path=/path/to/the/Antares/solver/antares-x.y-solver.exe ``` # TO DO