From 52076d2f59542b4500c543b2dcd64f7f3674619f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Lapr=C3=A9?= Date: Sat, 12 Oct 2024 21:52:57 +0200 Subject: [PATCH 1/5] First try at changing the logging facilities --- pyemu/utils/log_utils.py | 37 +++++++++++++++++++++++++++++++++++++ pyemu/utils/os_utils.py | 20 +++++++------------- 2 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 pyemu/utils/log_utils.py diff --git a/pyemu/utils/log_utils.py b/pyemu/utils/log_utils.py new file mode 100644 index 000000000..d10d247ce --- /dev/null +++ b/pyemu/utils/log_utils.py @@ -0,0 +1,37 @@ +"""Utilities to help with logging.""" + +import logging +import sys +from typing import Optional, Union + +FILE_LOGGER = logging.FileHandler("pyemu.log", delay=True) +STREAM_LOGGER = logging.StreamHandler(sys.stdout) + + +def get_logger( + name: Optional[str] = "pyemu", + verbose: bool = False, + logger: Union[bool, logging.Logger] = True, +) -> logging.Logger: + """Get a logger instance. + + Used to either get + Args: + name (`str`): name of the logger (default: "pyemu") + logger (`bool` or `logging.Logger`): either a boolean indicating to write to + "pyemu.log" or a logger to return as is. + + Returns: + logging.Logger object + """ + if isinstance(logger, bool): + create_file = logger + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + if create_file is True and FILE_LOGGER not in logger.handlers: + logger.addHandler(FILE_LOGGER) + if verbose and STREAM_LOGGER not in logger.handlers: + logger.addHandler(STREAM_LOGGER) + if not verbose and STREAM_LOGGER in logger.handlers: + logger.removeHandler(STREAM_LOGGER) + return logger diff --git a/pyemu/utils/os_utils.py b/pyemu/utils/os_utils.py index 69503ca4b..e107ebe59 100644 --- a/pyemu/utils/os_utils.py +++ b/pyemu/utils/os_utils.py @@ -11,6 +11,7 @@ import time from datetime import datetime from ..pyemu_warnings import PyemuWarning +from . import log_utils ext = "" bin_path = os.path.join("..", "bin") @@ -184,26 +185,19 @@ def run_sp(cmd_str, cwd=".", verbose=True, logfile=False, **kwargs): try: cmd_ins = [i for i in cmd_str.split()] - log_stream = open(os.path.join('pyemu.log'), 'w+', newline='') if logfile else None - with sp.Popen(cmd_ins, stdout=sp.PIPE, - stderr=sp.STDOUT, text=True, - shell=shell, bufsize=1) as process: + logger = log_utils.get_logger(verbose=verbose, logger=logfile) + with sp.Popen( + cmd_ins, stdout=sp.PIPE, stderr=sp.STDOUT, text=True, shell=shell, bufsize=1 + ) as process: for line in process.stdout: - if verbose: - print(line, flush=True, end='') - if logfile: - log_stream.write(line.strip('\n')) - log_stream.flush() - process.wait() # wait for the process to finish + logger.info(line) + process.wait() # wait for the process to finish retval = process.returncode except Exception as e: os.chdir(bwd) raise Exception("run() raised :{0}".format(str(e))) - finally: - if logfile: - log_stream.close() os.chdir(bwd) if "window" in platform.platform().lower(): From 90c12f3f8677f954b96baa88dac2e9abea8d81ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Lapr=C3=A9?= Date: Sun, 13 Oct 2024 12:36:34 +0200 Subject: [PATCH 2/5] feat: simplify get_logger, add set_logger --- pyemu/utils/log_utils.py | 43 +++++++++++++++++++++++++++------------- pyemu/utils/os_utils.py | 10 ++++++---- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/pyemu/utils/log_utils.py b/pyemu/utils/log_utils.py index d10d247ce..8f4915c98 100644 --- a/pyemu/utils/log_utils.py +++ b/pyemu/utils/log_utils.py @@ -2,16 +2,28 @@ import logging import sys -from typing import Optional, Union +from typing import Optional -FILE_LOGGER = logging.FileHandler("pyemu.log", delay=True) -STREAM_LOGGER = logging.StreamHandler(sys.stdout) +FILE_HANDLER = logging.FileHandler("pyemu.log", delay=True) +STREAM_HANDLER = logging.StreamHandler(sys.stdout) + +LOGGER: Optional[logging.Logger] = None + + +def set_logger(logger: logging.Logger) -> None: + """Set the global logger to be used by pyemu. + + Args: + logger (logging.Logger): the logger to be used. + """ + global LOGGER + LOGGER = logger def get_logger( name: Optional[str] = "pyemu", verbose: bool = False, - logger: Union[bool, logging.Logger] = True, + logfile: bool = False, ) -> logging.Logger: """Get a logger instance. @@ -24,14 +36,17 @@ def get_logger( Returns: logging.Logger object """ - if isinstance(logger, bool): - create_file = logger - logger = logging.getLogger(name) - logger.setLevel(logging.INFO) - if create_file is True and FILE_LOGGER not in logger.handlers: - logger.addHandler(FILE_LOGGER) - if verbose and STREAM_LOGGER not in logger.handlers: - logger.addHandler(STREAM_LOGGER) - if not verbose and STREAM_LOGGER in logger.handlers: - logger.removeHandler(STREAM_LOGGER) + if LOGGER is not None: + return LOGGER + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + _toggle_handler(logfile, logger, FILE_HANDLER) + _toggle_handler(verbose, logger, STREAM_HANDLER) return logger + + +def _toggle_handler(switch: bool, logger: logging.Logger, handler: logging.Handler): + if switch and handler not in logger.handlers: + logger.addHandler(handler) + if not switch and handler in logger.handlers: + logger.removeHandler(handler) diff --git a/pyemu/utils/os_utils.py b/pyemu/utils/os_utils.py index e107ebe59..b2896b7bf 100644 --- a/pyemu/utils/os_utils.py +++ b/pyemu/utils/os_utils.py @@ -174,8 +174,10 @@ def run_sp(cmd_str, cwd=".", verbose=True, logfile=False, **kwargs): # print warning if shell is True if shell: - warnings.warn("shell=True is not recommended and may cause issues, but hey! YOLO", PyemuWarning) - + warnings.warn( + "shell=True is not recommended and may cause issues, but hey! YOLO", + PyemuWarning, + ) bwd = os.getcwd() os.chdir(cwd) @@ -185,12 +187,12 @@ def run_sp(cmd_str, cwd=".", verbose=True, logfile=False, **kwargs): try: cmd_ins = [i for i in cmd_str.split()] - logger = log_utils.get_logger(verbose=verbose, logger=logfile) + logger = log_utils.get_logger(verbose=verbose, logfile=logfile) with sp.Popen( cmd_ins, stdout=sp.PIPE, stderr=sp.STDOUT, text=True, shell=shell, bufsize=1 ) as process: for line in process.stdout: - logger.info(line) + logger.info(line.rstrip("\n")) process.wait() # wait for the process to finish retval = process.returncode From 53d46fbad4236bc0d5221a85d8637b0cc4d1d1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Lapr=C3=A9?= Date: Sun, 13 Oct 2024 20:24:17 +0200 Subject: [PATCH 3/5] feat: add basicConfig parameter option --- pyemu/utils/log_utils.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pyemu/utils/log_utils.py b/pyemu/utils/log_utils.py index 8f4915c98..571c47c95 100644 --- a/pyemu/utils/log_utils.py +++ b/pyemu/utils/log_utils.py @@ -2,7 +2,7 @@ import logging import sys -from typing import Optional +from typing import Any, Optional FILE_HANDLER = logging.FileHandler("pyemu.log", delay=True) STREAM_HANDLER = logging.StreamHandler(sys.stdout) @@ -10,13 +10,30 @@ LOGGER: Optional[logging.Logger] = None -def set_logger(logger: logging.Logger) -> None: +def set_logger( + logger: Optional[logging.Logger] = None, + /, + **basic_config: Any, +) -> None: """Set the global logger to be used by pyemu. Args: logger (logging.Logger): the logger to be used. + **basic_config (Any): keyword arguments to `logging.basicConfig` """ global LOGGER + if logger is not None: + if basic_config: + raise ValueError( + "If a logger is passed no extra keyword arguments should be passed as well." + ) + else: + if not basic_config: + raise ValueError( + "If no logger is passed then keyword arguments for logging.basicConfig should be passed." + ) + logging.basicConfig(**basic_config) + logger = logging.getLogger("pyemu") LOGGER = logger From 4928996bdc7d7c1c13dd47af2275576613ddc5d0 Mon Sep 17 00:00:00 2001 From: Hugo Lapre Date: Mon, 4 Nov 2024 17:00:59 +0100 Subject: [PATCH 4/5] fix: allow easy control of the logfile location for testing --- autotest/utils_tests.py | 13 +++++++++---- pyemu/utils/log_utils.py | 20 +++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/autotest/utils_tests.py b/autotest/utils_tests.py index dd3a1f219..8d06d760d 100644 --- a/autotest/utils_tests.py +++ b/autotest/utils_tests.py @@ -2099,10 +2099,15 @@ def run_sp_capture_output_test(tmp_path): else: shell = False log_file = os.path.join(tmp_path, "pyemu.log") - pyemu.os_utils.run("echo Hello World", - verbose=False, use_sp=True, - shell=shell, cwd=tmp_path, logfile=True) - + pyemu.os_utils.run( + "echo Hello World", + verbose=False, + use_sp=True, + shell=shell, + cwd=tmp_path, + logfile=log_file, + ) + with open(log_file, 'r') as f: content = f.read() assert "Hello World" in content diff --git a/pyemu/utils/log_utils.py b/pyemu/utils/log_utils.py index 571c47c95..82c59753a 100644 --- a/pyemu/utils/log_utils.py +++ b/pyemu/utils/log_utils.py @@ -40,7 +40,7 @@ def set_logger( def get_logger( name: Optional[str] = "pyemu", verbose: bool = False, - logfile: bool = False, + logfile: bool | str = False, ) -> logging.Logger: """Get a logger instance. @@ -57,13 +57,15 @@ def get_logger( return LOGGER logger = logging.getLogger(name) logger.setLevel(logging.INFO) - _toggle_handler(logfile, logger, FILE_HANDLER) - _toggle_handler(verbose, logger, STREAM_HANDLER) - return logger - -def _toggle_handler(switch: bool, logger: logging.Logger, handler: logging.Handler): - if switch and handler not in logger.handlers: - logger.addHandler(handler) - if not switch and handler in logger.handlers: + for handler in logger.handlers: logger.removeHandler(handler) + if logfile is True: + logger.addHandler(FILE_HANDLER) + elif isinstance(logfile, str): + logger.addHandler(logging.FileHandler(logfile)) + + if verbose: + logger.addHandler(STREAM_HANDLER) + + return logger From 4ed389549a2e9d9ae52bb5fa29acd815aa49e179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Lapr=C3=A9?= Date: Wed, 6 Nov 2024 09:03:42 +0100 Subject: [PATCH 5/5] fix: correct union syntax for older python versions --- pyemu/utils/log_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyemu/utils/log_utils.py b/pyemu/utils/log_utils.py index 82c59753a..646eb40d1 100644 --- a/pyemu/utils/log_utils.py +++ b/pyemu/utils/log_utils.py @@ -2,7 +2,7 @@ import logging import sys -from typing import Any, Optional +from typing import Any, Optional, Union FILE_HANDLER = logging.FileHandler("pyemu.log", delay=True) STREAM_HANDLER = logging.StreamHandler(sys.stdout) @@ -40,7 +40,7 @@ def set_logger( def get_logger( name: Optional[str] = "pyemu", verbose: bool = False, - logfile: bool | str = False, + logfile: Union[bool , str] = False, ) -> logging.Logger: """Get a logger instance.