Skip to content

Commit

Permalink
refactor(log_parser): simplify implementation of log parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
laurent-laporte-pro committed Aug 2, 2023
1 parent 38d6bea commit 2b3878b
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 45 deletions.
7 changes: 2 additions & 5 deletions antarest/launcher/adapters/abstractlauncher.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
)
from antarest.core.model import PermissionInfo, PublicMode
from antarest.core.requests import RequestParameters
from antarest.launcher.adapters.log_parser import LaunchProgressDTO, LogParser
from antarest.launcher.adapters.log_parser import LaunchProgressDTO
from antarest.launcher.model import JobStatus, LauncherParametersDTO, LogType


Expand Down Expand Up @@ -104,10 +104,7 @@ def update_log(log_line: str) -> None:
)
progress_updated = False
for line in log_line.split("\n"):
progress_updated = (
LogParser.update_progress(line, launch_progress_dto)
or progress_updated
)
progress_updated |= launch_progress_dto.update_progress(line)
if progress_updated:
self.event_bus.push(
Event(
Expand Down
22 changes: 6 additions & 16 deletions antarest/launcher/adapters/log_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,42 +3,32 @@

from pydantic import BaseModel


logger = logging.getLogger(__name__)


class LaunchProgressDTO(BaseModel):
progress: float = 0
total_mc_years_to_perform: int = 1

total_mc_years: int = 1

class LogParser:
@staticmethod
def update_progress(
line: str, launch_progress_dto: LaunchProgressDTO
) -> bool:
def update_progress(self, line: str) -> bool:
if "MC-Years : [" in line:
if regex_result := re.search(
r"MC-Years : \[\d+ .. \d+], total: (\d+)", line
):
launch_progress_dto.total_mc_years_to_perform = int(
regex_result[1]
)
self.total_mc_years = int(regex_result[1])
return True
else:
logger.warning(
f"Failed to extract log progress batch size on line : {line}"
)
return False
elif "Exporting the annual results" in line:
launch_progress_dto.progress += (
98 / launch_progress_dto.total_mc_years_to_perform
)
self.progress += 98 / self.total_mc_years
return True
elif "Exporting the survey results" in line:
launch_progress_dto.progress = 99
self.progress = 99
return True
elif "Quitting the solver gracefully" in line:
launch_progress_dto.progress = 100
self.progress = 100
return True
return False
44 changes: 20 additions & 24 deletions tests/launcher/test_log_parser.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,38 @@
import pytest
from tests.storage.integration.data import simulation_log


from antarest.launcher.adapters.log_parser import LaunchProgressDTO, LogParser
from antarest.launcher.adapters.log_parser import LaunchProgressDTO
from tests.storage.integration.data import simulation_log


@pytest.mark.parametrize(
"launch_progress_dto,line,expected_progression,expected_output",
[
(
LaunchProgressDTO(total_mc_years_to_perform=100),
LaunchProgressDTO(total_mc_years=100),
"[infos] MC-Years : [1 .. 11], total: 11",
0,
True,
),
(
LaunchProgressDTO(total_mc_years_to_perform=10),
LaunchProgressDTO(total_mc_years=10),
"this is a test",
0,
False,
),
(
LaunchProgressDTO(total_mc_years_to_perform=100),
LaunchProgressDTO(total_mc_years=100),
"[solver][infos] parallel batch size : 10",
0,
False,
),
(
LaunchProgressDTO(total_mc_years_to_perform=10),
LaunchProgressDTO(total_mc_years=10),
"[solver][infos] Exporting the annual results",
9.8,
True,
),
(
LaunchProgressDTO(total_mc_years_to_perform=10),
LaunchProgressDTO(total_mc_years=10),
"[solver][infos] Exporting the survey results",
99,
True,
Expand All @@ -46,35 +45,32 @@ def test_update_progress(
expected_progression: float,
expected_output: bool,
):
output = LogParser.update_progress(line, launch_progress_dto)
output = launch_progress_dto.update_progress(line)
assert launch_progress_dto.progress == expected_progression
assert output == expected_output


def test_update_progress_with_real_log():
real_log = simulation_log.simulation_log.split("\n")
launch_progress_dto = LaunchProgressDTO()
dto = LaunchProgressDTO()
for line in real_log:
if "Exporting the annual results" in line:
pre_update_progress = launch_progress_dto.progress
LogParser.update_progress(line, launch_progress_dto)
pre_update_progress = dto.progress
dto.update_progress(line)
assert (
launch_progress_dto.progress
== pre_update_progress
+ 98 / launch_progress_dto.total_mc_years_to_perform
dto.progress == pre_update_progress + 98 / dto.total_mc_years
)
continue
elif "Exporting the survey results" in line:
pre_update_progress = launch_progress_dto.progress
pre_update_progress = dto.progress
assert pre_update_progress < 99
LogParser.update_progress(line, launch_progress_dto)
assert launch_progress_dto.progress == 99
dto.update_progress(line)
assert dto.progress == 99
continue
elif "Quitting the solver gracefully" in line:
assert launch_progress_dto.progress == 99
LogParser.update_progress(line, launch_progress_dto)
assert launch_progress_dto.progress == 100
assert dto.progress == 99
dto.update_progress(line)
assert dto.progress == 100
continue
LogParser.update_progress(line, launch_progress_dto)
print(launch_progress_dto.progress)
assert launch_progress_dto.total_mc_years_to_perform == 2
dto.update_progress(line)
assert dto.total_mc_years == 2

0 comments on commit 2b3878b

Please sign in to comment.