Add cucumber tests tool (#2347)

.github/workflows/cucumber-tests/action.yml

@@ -0,0 +1,23 @@
+name: "Run cucumber tests"
+description: "Run cucumber tests"
+inputs:
+  feature:
+    description: 'Feature file or folder to run (default runs all features in "features" folder)'
+    required: false
+    default: 'features'
+  tags:
+    description: 'Tags to run (default skips tests marked @flaky)'
+    required: false
+    default: '~@flaky'
+runs:
+  using: "composite"
+  steps:
+    - name: Install Python requirements
+      shell: bash
+      run: python3 -m pip install -r src/tests/cucumber/requirements.txt
+
+    - name: Run tests
+      shell: bash
+      run: |
+        cd src/tests/cucumber
+        behave --tags ${{ inputs.tags }} ${{ inputs.feature }}

.github/workflows/ubuntu.yml

@@ -142,10 +142,14 @@ jobs:
         run: |
           echo "SIMTEST=${{ fromJson(env.SIMTEST_JSON).version }}" >> $GITHUB_ENV
 
-      - name: Init submodule
+      - name: Init submodule Antares_Simulator_Tests
         run: |
           git submodule update --init --remote --recursive src/tests/resources/Antares_Simulator_Tests
 
+      - name: Init submodule Antares_Simulator_Tests_NR
+        run: |
+          git submodule update --init --remote --recursive src/tests/resources/Antares_Simulator_Tests_NR
+
       - name: Run named mps tests
         if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }}
         uses: ./.github/workflows/run-tests
@@ -231,6 +235,11 @@ jobs:
           batch-name: short-tests
           os: ${{ env.os }}
 
+      - name: Run cucumber on short-tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/short_tests.feature"
+
       - name: Run mps tests
         if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }}
         uses: ./.github/workflows/run-tests
@@ -273,6 +282,11 @@ jobs:
           batch-name: medium-tests
           os: ${{ env.os }}
 
+      - name: Run cucumber on medium-tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/medium_tests.feature"
+
       - name: Run long-tests-1
         if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }}
         uses: ./.github/workflows/run-tests

.github/workflows/windows-vcpkg.yml

@@ -110,10 +110,14 @@ jobs:
       - name: Install pip dependencies if necessary
         run: pip install -r src/tests/examples/requirements.txt
 
-      - name: Init submodule
+      - name: Init submodule Antares_Simulator_Tests
         run: |
           git submodule update --init --remote src/tests/resources/Antares_Simulator_Tests
 
+      - name: Init submodule Antares_Simulator_Tests_NR
+        run: |
+          git submodule update --init --remote src/tests/resources/Antares_Simulator_Tests_NR
+
       - name: Enable git longpaths
         run: git config --system core.longpaths true
 
@@ -241,6 +245,11 @@ jobs:
           batch-name: short-tests
           os: ${{ env.os }}
 
+      - name: Run cucumber on short-tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/short_tests.feature"
+
       - name: Run mps tests
         if: ${{ env.RUN_SIMPLE_TESTS == 'true' && !cancelled() }}
         uses: ./.github/workflows/run-tests
@@ -275,6 +284,11 @@ jobs:
           batch-name: medium-tests
           os: ${{ env.os }}
 
+      - name: Run cucumber on medium-tests
+        uses: ./.github/workflows/cucumber-tests
+        with:
+          feature: "features/medium_tests.feature"
+
       - name: Run long-tests-1
         if: ${{ env.RUN_EXTENDED_TESTS == 'true' && !cancelled() }}
         uses: ./.github/workflows/run-tests

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "vcpkg"]
 	path = vcpkg
 	url = https://github.com/microsoft/vcpkg.git
+[submodule "src/tests/resources/Antares_Simulator_Tests_NR"]
+	path = src/tests/resources/Antares_Simulator_Tests_NR
+	url = https://github.com/AntaresSimulatorTeam/Antares_Simulator_Tests_NR.git

src/tests/CMakeLists.txt

@@ -9,8 +9,7 @@ if (Boost_FOUND)
     set_target_properties(Boost::unit_test_framework PROPERTIES IMPORTED_GLOBAL TRUE)
 endif()
 
-
-
+add_subdirectory(cucumber)
 
 # end to end test require boost 1.6.x because of boost::test_tools::tolerance, BOOST_TEST
 # old versions of Boost don't contain a '.', also ignore them

src/tests/cucumber/CMakeLists.txt

@@ -0,0 +1,2 @@
+file(GENERATE OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/conf.yaml CONTENT "antares-solver : $<TARGET_FILE:antares-solver>"
+        CONDITION $<STREQUAL:$<CONFIG>,${CMAKE_BUILD_TYPE}>)

src/tests/cucumber/features/medium_tests.feature

@@ -0,0 +1,15 @@
+Feature: medium tests
+
+  @fast @medium @incomplete
+  Scenario: 035 Mixed Expansion - Smart grid model 2
+    Given the study path is "medium-tests/035 Mixed Expansion - Smart grid model 2"
+    When I run antares simulator
+    Then the simulation takes less than 15 seconds
+    And the simulation succeeds
+    And the expected value of the annual system cost is 3.725e+10
+    And the minimum annual system cost is 3.642e+10
+    And the maximum annual system cost is 4.011e+10
+    And the annual system cost is
+      | EXP       | STD       | MIN       | MAX       |
+      | 3.725e+10 | 1.063e+09 | 3.642e+10 | 4.011e+10 |
+    # TODO : add steps when we understand what this test is supposed to test

src/tests/cucumber/features/short_tests.feature

@@ -0,0 +1,45 @@
+Feature: short tests
+
+  @fast @short
+  Scenario: 001 One node - passive
+    Given the study path is "short-tests/001 One node - passive"
+    When I run antares simulator
+    Then the simulation takes less than 5 seconds
+    And the simulation succeeds
+    And the annual system cost is
+      | EXP | STD | MIN | MAX |
+      | 0   | 0   | 0   | 0   |
+
+  @fast @short
+  Scenario: 002 Thermal fleet - Base
+    Given the study path is "short-tests/002 Thermal fleet - Base"
+    When I run antares simulator
+    Then the simulation takes less than 5 seconds
+    And the simulation succeeds
+    And the annual system cost is
+      | EXP      | STD | MIN      | MAX      |
+      | 2.729e+7 | 0   | 2.729e+7 | 2.729e+7 |
+    And in area "AREA", during year 1, loss of load lasts 1 hours
+    And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW
+
+  @fast @short
+  Scenario: 003 Thermal fleet - Must-run
+    Given the study path is "short-tests/003 Thermal fleet - Must-run"
+    When I run antares simulator
+    Then the simulation takes less than 5 seconds
+    And the simulation succeeds
+    And the annual system cost is
+      | EXP      | STD | MIN      | MAX      |
+      | 2.751e+7 | 0   | 2.751e+7 | 2.751e+7 |
+    And in area "AREA", during year 1, loss of load lasts 1 hours
+    And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW
+
+  @fast @short
+  Scenario: 021 Four areas - DC law
+    Given the study path is "short-tests/021 Four areas - DC law"
+    When I run antares simulator
+    Then the simulation takes less than 20 seconds
+    And the simulation succeeds
+    And the annual system cost is
+      | EXP       | STD       | MIN       | MAX       |
+      | 7.972e+10 | 2.258e+10 | 5.613e+10 | 1.082e+11 |

src/tests/cucumber/features/steps/assertions.py

@@ -0,0 +1,4 @@
+# Custom assertions
+
+def assert_double_close(expected, actual, relative_tolerance):
+    assert abs((actual - expected) / max(1e-6, expected)) <= relative_tolerance

src/tests/cucumber/features/steps/context_utils.py

@@ -0,0 +1,22 @@
+# Manage cached output data in "context" object
+
+from output_utils import *
+
+def get_annual_system_cost(context):
+    if context.annual_system_cost is None:
+        context.annual_system_cost = parse_annual_system_cost(context.output_path)
+    return context.annual_system_cost
+
+def get_hourly_values_for_specific_hour(context, area : str, year : int, date : str):
+    df = get_hourly_values(context, area, year)
+    day, month, hour = date.split(" ")
+    return df.loc[(df['Unnamed: 2'] == int(day)) & (df['Unnamed: 3'] == month) & (df['Unnamed: 4'] == hour)]
+
+def get_hourly_values(context, area : str, year : int):
+    if context.hourly_values is None:
+        context.hourly_values = {}
+    if area not in context.hourly_values:
+        context.hourly_values[area] = {}
+    if year not in context.hourly_values[area]:
+        context.hourly_values[area][year] = parse_hourly_values(context.output_path, area, year)
+    return context.hourly_values[area][year]

src/tests/cucumber/features/steps/output_utils.py

@@ -0,0 +1,37 @@
+# Antares outputs parsing
+
+import os
+import pandas
+import configparser
+
+def parse_output_folder_from_logs(logs: bytes) -> str:
+    for line in logs.splitlines():
+        if b'Output folder : ' in line:
+            return line.split(b'Output folder : ')[1].decode('ascii')
+    raise LookupError("Could not parse output folder in output logs")
+
+
+def parse_annual_system_cost(output_path: str) -> dict:
+    file = open(os.path.join(output_path, "annualSystemCost.txt"), 'r')
+    keys = ["EXP", "STD", "MIN", "MAX"]
+    annual_system_cost = {}
+    for line in file.readlines():
+        for key in keys:
+            if key in line:
+                annual_system_cost[key] = float(line.split(key + " : ")[1])
+    return annual_system_cost
+
+
+def parse_simu_time(output_path: str) -> float:
+    execution_info = configparser.ConfigParser()
+    execution_info.read(os.path.join(output_path, "execution_info.ini"))
+    return float(execution_info['durations_ms']['total']) / 1000
+
+
+def parse_hourly_values(output_path: str, area: str, year: int):
+    return read_csv(os.path.join(output_path, "economy", "mc-ind", f"{year:05d}", "areas", area, "values-hourly.txt"))
+
+
+def read_csv(file_name):
+    ignore_rows = [0, 1, 2, 3, 5, 6]
+    return pandas.read_csv(file_name, skiprows=ignore_rows, sep='\t', low_memory=False)

src/tests/cucumber/features/steps/simulator_utils.py

@@ -0,0 +1,47 @@
+# Methods to run Antares simulator
+
+import subprocess
+import glob
+import yaml
+from pathlib import Path
+from study_input_handler import study_input_handler
+from output_utils import parse_output_folder_from_logs
+
+
+def get_solver_path():
+    with open("conf.yaml") as file:
+            content = yaml.full_load(file)
+    return content.get("antares-solver")
+
+SOLVER_PATH = get_solver_path() # we only need to run this once
+
+
+def run_simulation(context):
+    activate_simu_outputs(context)  # TODO : remove this and update studies instead
+    command = build_antares_solver_command(context)
+    print(f"Running command: {command}")
+    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    out, err = process.communicate()
+    context.output_path = parse_output_folder_from_logs(out)
+    context.return_code = process.returncode
+    context.annual_system_cost = None
+    context.hourly_values = None
+
+
+def activate_simu_outputs(context):
+    sih = study_input_handler(Path(context.study_path))
+    sih.set_value(variable="synthesis", value="true", file_nick_name="general")
+    sih.set_value(variable="year-by-year", value="true", file_nick_name="general")
+
+
+def build_antares_solver_command(context):
+    command = [SOLVER_PATH, "-i", str(context.study_path)]
+    if context.use_ortools:
+        command.append('--use-ortools')
+        command.append('--ortools-solver=' + context.ortools_solver)
+    if context.named_mps_problems:
+        command.append('--named-mps-problems')
+    if context.parallel:
+        command.append('--force-parallel=4')
+    return command
+

src/tests/cucumber/features/steps/steps.py

@@ -0,0 +1,67 @@
+# Gherkins test steps definitions
+
+import os
+from behave import *
+from simulator_utils import *
+from assertions import *
+from context_utils import *
+
+@given('the study path is "{string}"')
+def study_path_is(context, string):
+    context.study_path = os.path.join("..", "resources", "Antares_Simulator_Tests_NR" , string.replace("/", os.sep))
+
+@when('I run antares simulator')
+def run_antares(context):
+    context.use_ortools = True
+    context.ortools_solver = "sirius"
+    context.named_mps_problems = False
+    context.parallel = False
+    run_simulation(context)
+
+@then('the simulation succeeds')
+def simu_success(context):
+    assert context.return_code == 0
+
+@then('the simulation fails')
+def simu_success(context):
+    assert context.return_code != 0
+
+@then('the expected value of the annual system cost is {value}')
+def check_annual_cost_expected(context, value):
+    assert_double_close(float(value), get_annual_system_cost(context)["EXP"], 0.001)
+
+@then('the minimum annual system cost is {value}')
+def check_annual_cost_min(context, value):
+    assert_double_close(float(value), get_annual_system_cost(context)["MIN"], 0.001)
+
+@then('the maximum annual system cost is {value}')
+def check_annual_cost_max(context, value):
+    assert_double_close(float(value), get_annual_system_cost(context)["MAX"], 0.001)
+
+@then('the annual system cost is')
+def check_annual_cost(context):
+    for row in context.table:
+        assert_double_close(float(row["EXP"]), get_annual_system_cost(context)["EXP"], 0.001)
+        assert_double_close(float(row["STD"]), get_annual_system_cost(context)["STD"], 0.001)
+        assert_double_close(float(row["MIN"]), get_annual_system_cost(context)["MIN"], 0.001)
+        assert_double_close(float(row["MAX"]), get_annual_system_cost(context)["MAX"], 0.001)
+
+@then('the simulation takes less than {seconds} seconds')
+def check_simu_time(context, seconds):
+    actual_simu_time = parse_simu_time(context.output_path)
+    assert actual_simu_time <= float(seconds)
+
+@then('in area "{area}", during year {year}, loss of load lasts {lold_hours} hours')
+def check_lold_duration(context, area, year, lold_hours):
+    assert int(lold_hours) == get_hourly_values(context, area.lower(), int(year))["LOLD"].sum()
+
+@then('in area "{area}", unsupplied energy on "{date}" of year {year} is of {lold_value_mw} MW')
+def check_lold_value(context, area, date, year, lold_value_mw):
+    actual_unsp_energ = get_hourly_values_for_specific_hour(context, area.lower(), int(year), date)["UNSP. ENRG"].sum()
+    assert_double_close(float(lold_value_mw), actual_unsp_energ, 0.001)
+
+def after_feature(context, feature):
+    # post-processing a test: clean up output files to avoid taking up all the disk space
+    if (context.output_path != None):
+        pathlib.Path.rmdir(context.output_path)
+