diff --git a/changelog/fragments/1469.changed.rst b/changelog/fragments/1469.changed.rst new file mode 100644 index 0000000000000000000000000000000000000000..82f7792678251b9a305ab85cf1e56b48d3e04aa8 --- /dev/null +++ b/changelog/fragments/1469.changed.rst @@ -0,0 +1 @@ +Improvement of RetryDiscipline performance when timeout option is activated. diff --git a/pyproject.toml b/pyproject.toml index dcdd2e7a964538b6d90ee74b755280b176c24f33..eebef4f66e0dc35f95756b64952ff25466d65ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ dynamic = ["version"] requires-python = ">=3.9, <3.13" dependencies = [ + "PyThreadKiller >= 3.0.6", "docstring-inheritance >=1.0.0,<=2.2.2", # For supporting new type annotations in pydantic models. "eval-type-backport==0.2.0 ; python_version<'3.10'", diff --git a/requirements/check.txt b/requirements/check.txt index ad9323ef1498bdc574fef4ea4e11eb9134d04b4f..80073a9415cd6786b77537a462466deb85e7dae8 100644 --- a/requirements/check.txt +++ b/requirements/check.txt @@ -4,15 +4,15 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.9 # via virtualenv -filelock==3.17.0 +filelock==3.16.1 # via virtualenv -identify==2.6.8 +identify==2.6.1 # via pre-commit nodeenv==1.9.1 # via pre-commit platformdirs==4.3.6 # via virtualenv -pre-commit==4.1.0 +pre-commit==3.5.0 # via -r requirements/check.in pyyaml==6.0.2 # via pre-commit diff --git a/requirements/dist.txt b/requirements/dist.txt index e210a5eb7b2b1810bd95aaca5225fe083a63d062..b7766b06246113cb24be993716b40e5c7df6451d 100644 --- a/requirements/dist.txt +++ b/requirements/dist.txt @@ -4,6 +4,8 @@ annotated-types==0.7.0 # via pydantic attrs==25.1.0 # via check-wheel-contents +backports-tarfile==1.2.0 + # via jaraco-context build==1.2.2.post1 # via -r requirements/dist.in certifi==2025.1.31 @@ -18,12 +20,19 @@ click==8.1.8 # via check-wheel-contents cryptography==44.0.2 # via secretstorage -docutils==0.21.2 +docutils==0.20.1 # via readme-renderer id==1.5.0 # via twine idna==3.10 # via requests +importlib-metadata==8.5.0 + # via + # build + # keyring + # twine +importlib-resources==6.4.5 + # via keyring jaraco-classes==3.4.0 # via keyring jaraco-context==6.0.1 @@ -34,13 +43,13 @@ jeepney==0.9.0 # via # keyring # secretstorage -keyring==25.6.0 +keyring==25.5.0 # via twine markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -more-itertools==10.6.0 +more-itertools==10.5.0 # via # jaraco-classes # jaraco-functools @@ -63,7 +72,7 @@ pygments==2.19.1 # rich pyproject-hooks==1.2.0 # via build -readme-renderer==44.0 +readme-renderer==43.0 # via twine requests==2.32.3 # via @@ -78,15 +87,25 @@ rich==13.9.4 # via twine secretstorage==3.3.3 # via keyring +tomli==2.2.1 + # via + # build + # check-wheel-contents twine==6.1.0 # via -r requirements/dist.in typing-extensions==4.12.2 # via + # annotated-types # pydantic # pydantic-core -urllib3==2.3.0 + # rich +urllib3==2.2.3 # via # requests # twine wheel-filename==1.4.2 # via check-wheel-contents +zipp==3.20.2 + # via + # importlib-metadata + # importlib-resources diff --git a/requirements/doc.txt b/requirements/doc.txt index d453ed8460540f0550deaaf20e8a733db04470e2..5fce0b8da61aad6f509f5ae77b16ebc2ff4a6998 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -152,6 +152,8 @@ python-dateutil==2.9.0.post0 # pandas python-dotenv==1.0.1 # via pydantic-settings +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.3.1 @@ -161,6 +163,7 @@ pyyaml==6.0.2 requests==2.32.3 # via # gemseo (pyproject.toml) + # pythreadkiller # sphinx # sphinxcontrib-spelling roman-numerals-py==3.1.0 diff --git a/requirements/test-python3.10.txt b/requirements/test-python3.10.txt index 0edcb79ad45975c46386557cc820257751c9b82a..f308f1bc6cfa450166f235d82adf76c700b460b1 100644 --- a/requirements/test-python3.10.txt +++ b/requirements/test-python3.10.txt @@ -123,12 +123,16 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.3.1 # via gemseo (pyproject.toml) requests==2.32.3 - # via gemseo (pyproject.toml) + # via + # gemseo (pyproject.toml) + # pythreadkiller scikit-learn==1.6.1 # via gemseo (pyproject.toml) scipy==1.15.2 diff --git a/requirements/test-python3.11.txt b/requirements/test-python3.11.txt index 6eda6185d92d8348618841c9dea4cce62d2e002e..9f7cd3e4b76076249b4b6a7c8840f9bf49e4a8bd 100644 --- a/requirements/test-python3.11.txt +++ b/requirements/test-python3.11.txt @@ -121,12 +121,16 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.3.1 # via gemseo (pyproject.toml) requests==2.32.3 - # via gemseo (pyproject.toml) + # via + # gemseo (pyproject.toml) + # pythreadkiller scikit-learn==1.6.1 # via gemseo (pyproject.toml) scipy==1.15.2 diff --git a/requirements/test-python3.12.txt b/requirements/test-python3.12.txt index c6d7576c08194d8639a32961354ee9c115869cd5..c1dbb6b0ccb78417fb420fd7ff1a0087d9a1d170 100644 --- a/requirements/test-python3.12.txt +++ b/requirements/test-python3.12.txt @@ -121,12 +121,16 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.3.1 # via gemseo (pyproject.toml) requests==2.32.3 - # via gemseo (pyproject.toml) + # via + # gemseo (pyproject.toml) + # pythreadkiller scikit-learn==1.6.1 # via gemseo (pyproject.toml) scipy==1.15.2 diff --git a/requirements/test-python3.9-min-deps-versions.txt b/requirements/test-python3.9-min-deps-versions.txt index 4949200d010bae80569fadefb2e96b56b5d66862..f9ee17b1e1a6373ec5d266d2146b014ae8e0313e 100644 --- a/requirements/test-python3.9-min-deps-versions.txt +++ b/requirements/test-python3.9-min-deps-versions.txt @@ -6,6 +6,10 @@ attrs==25.1.0 # via pytest cached-property==2.0.1 # via h5py +certifi==2025.1.31 + # via requests +charset-normalizer==3.4.1 + # via requests covdefaults==0.1.0 # via gemseo (pyproject.toml) coverage==7.6.12 @@ -34,6 +38,8 @@ graphviz==0.16 # via gemseo (pyproject.toml) h5py==3.0.0 # via gemseo (pyproject.toml) +idna==3.10 + # via requests importlib-metadata==3.6.0 # via gemseo (pyproject.toml) iniconfig==2.0.0 @@ -117,12 +123,16 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.1.0 # via gemseo (pyproject.toml) -requests==0.2.0 - # via gemseo (pyproject.toml) +requests==2.31.0 + # via + # gemseo (pyproject.toml) + # pythreadkiller schema==0.7.7 # via pandera scikit-learn==1.0 @@ -155,6 +165,8 @@ typing-extensions==4.6.1 # gemseo (pyproject.toml) # pydantic # pydantic-core +urllib3==2.3.0 + # via requests wcwidth==0.2.13 # via prettytable wrapt==1.17.2 diff --git a/requirements/test-python3.9.txt b/requirements/test-python3.9.txt index e2316b88713bf9d06944d5f88155fa1c03b31fa3..9239bd9d7e16416c07f4c964c16a1462fc6117c1 100644 --- a/requirements/test-python3.9.txt +++ b/requirements/test-python3.9.txt @@ -131,12 +131,16 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas +pythreadkiller==3.0.6 + # via gemseo (pyproject.toml) pytz==2025.1 # via pandas pyxdsm==2.3.1 # via gemseo (pyproject.toml) requests==2.32.3 - # via gemseo (pyproject.toml) + # via + # gemseo (pyproject.toml) + # pythreadkiller scikit-learn==1.6.1 # via gemseo (pyproject.toml) scipy==1.13.1 diff --git a/src/gemseo/disciplines/wrappers/retry_discipline.py b/src/gemseo/disciplines/wrappers/retry_discipline.py index cdcd21a828216184bc644b88fb37c1af652a8314..87a457f130c50e398080fc279288c6c3068b55c0 100644 --- a/src/gemseo/disciplines/wrappers/retry_discipline.py +++ b/src/gemseo/disciplines/wrappers/retry_discipline.py @@ -18,16 +18,15 @@ from __future__ import annotations import concurrent.futures as cfutures import math -import os -import signal import time -from concurrent.futures import ProcessPoolExecutor from logging import getLogger from typing import TYPE_CHECKING from typing import ClassVar from gemseo.core.discipline import Discipline from gemseo.core.execution_status import ExecutionStatus +from gemseo.utils.chrono import Chrono +from gemseo.utils.thread_killer import ThreadKiller if TYPE_CHECKING: from collections.abc import Iterable @@ -194,30 +193,35 @@ class RetryDiscipline(Discipline): self.timeout, ) - with ProcessPoolExecutor() as executor: - run_discipline = executor.submit( - self._discipline.execute, - input_data, - ) - - try: - return run_discipline.result(timeout=self.timeout) - - except self.__time_out_exceptions: - # Killing the children is mandatory to abort the discipline execution - # immediately: shutdown + kill children. - pid_child = [p.pid for p in executor._processes.values()] - executor.shutdown(wait=False, cancel_futures=True) - - LOGGER.debug("killing subprocesses: %s", pid_child) - for pid in pid_child: - os.kill(pid, signal.SIGTERM) + thread = ThreadKiller( + target=self._discipline.execute, + kwargs={"input_data": input_data}, + ) - LOGGER.exception( - "Process stopped as it exceeds timeout (%s s)", self.timeout - ) - raise + try: + chrono = Chrono() + chrono.start_chrono() + thread.start() + + # Either thread terminates, raises an exception or the timeout is reached + go = True + wait_loop = 1.0e-3 + while go: + if chrono.elapsed_time() > self.timeout: + thread.kill() + raise TimeoutError # noqa: TRY301 + if not thread.is_alive(): + go = False + time.sleep(wait_loop) + + return thread.join() + + except self.__time_out_exceptions: + LOGGER.exception( + "Process stopped as it exceeds timeout (%s s)", self.timeout + ) + raise - except Exception as error: # noqa: BLE001 - LOGGER.debug(type(error)) - raise + except Exception as error: # noqa: BLE001 + LOGGER.debug(type(error)) + raise diff --git a/src/gemseo/utils/chrono.py b/src/gemseo/utils/chrono.py new file mode 100755 index 0000000000000000000000000000000000000000..470ea9b588e2f09f46db02c289bda793ca2002f6 --- /dev/null +++ b/src/gemseo/utils/chrono.py @@ -0,0 +1,35 @@ +# Copyright 2021 IRT Saint Exupéry, https://www.irt-saintexupery.com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""A class to measure the duration of a task.""" + +from __future__ import annotations + +import time + + +class Chrono: + """A class to measure the duration of a task.""" + + def __init__(self): + """Instantiation of the class.""" + self.start = None + + def start_chrono(self): + """Initialise the chronometer by storing the present time.""" + self.start = time.perf_counter() + + def elapsed_time(self): + """Computes the time elapsed from the start of the chronometer.""" + return time.perf_counter() - self.start diff --git a/src/gemseo/utils/thread_killer.py b/src/gemseo/utils/thread_killer.py new file mode 100755 index 0000000000000000000000000000000000000000..cc94e769a23925e08cfbfa053055876f38263f51 --- /dev/null +++ b/src/gemseo/utils/thread_killer.py @@ -0,0 +1,54 @@ +# Copyright 2021 IRT Saint Exupéry, https://www.irt-saintexupery.com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""ThreadKiller permits to kill a thread.""" + +from __future__ import annotations + +from PyThreadKiller import PyThreadKiller + + +class ThreadKiller(PyThreadKiller): + """Custom Thread class, inheritance of the PyThreadKiller class. + + This class adds the ability to capture an exception if any occurs. If the run + completes, then ``_result`` is returned through the ``join`` method. + """ + + _return = None + """The result returned by the thread once the run is completed.""" + + raised_exception = None + """The exception raised by the thread, if any encountered.""" + + def __init__(self, target=None, name=None, args=(), kwargs=None, *, daemon=None): + """Instantiate the class, with an inheritance of the PyThreadKiller class.""" + super().__init__( + target=target, name=name, args=args, kwargs=kwargs, daemon=daemon + ) + + def run(self): + """Run the thread and capture an exception if any.""" + try: + if self._target is not None: + self._return = self._target(*self._args, **self._kwargs) + except BaseException as raised_exception: # noqa: BLE001 + self.raised_exception = raised_exception + + def join(self, *args): + """Get the final result or raise an exception.""" + super().join(*args) + if self.raised_exception: + raise self.raised_exception + return self._return diff --git a/tests/utils/test_chrono.py b/tests/utils/test_chrono.py new file mode 100644 index 0000000000000000000000000000000000000000..3d10908ce9b9fe5b04bdae5265de4b945f84f8f4 --- /dev/null +++ b/tests/utils/test_chrono.py @@ -0,0 +1,31 @@ +# Copyright 2021 IRT Saint Exupéry, https://www.irt-saintexupery.com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""Test for the chrono.""" + +from __future__ import annotations + +from time import sleep + +import pytest + +from gemseo.utils.chrono import Chrono + + +def test_elapsed_time() -> None: + """Check the elapsed_time attribute.""" + chrono = Chrono() + chrono.start_chrono() + sleep(0.5) + assert chrono.elapsed_time() == pytest.approx(0.5, abs=0.1) diff --git a/tests/utils/test_thread_killer.py b/tests/utils/test_thread_killer.py new file mode 100644 index 0000000000000000000000000000000000000000..3ca16e65ae560711d853429832605efabe367ea6 --- /dev/null +++ b/tests/utils/test_thread_killer.py @@ -0,0 +1,46 @@ +# Copyright 2021 IRT Saint Exupéry, https://www.irt-saintexupery.com +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 3 as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""Tests for the thread killer.""" + +from __future__ import annotations + +from time import sleep + +from gemseo.utils.thread_killer import ThreadKiller + + +def sleeping_target() -> object: + """A target waiting one second before ending.""" + sleep(0.25) + return "Hello world" + + +def test_kill_thread() -> None: + """Kill a running thread""" + thread = ThreadKiller(target=sleeping_target) + thread.start() + sleep(0.05) + assert thread.is_alive() is True + thread.kill() + sleep(0.3) + assert isinstance(thread.raised_exception, SystemExit) is True + assert thread.is_alive() is False + + +def test_thread_start_and_join() -> None: + """Wait the thread ending and get the output.""" + thread = ThreadKiller(target=sleeping_target) + thread.start() + assert thread.join() == "Hello world"