From 8f3921a650666b0222de322e01644337bd64ea16 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 16 Jul 2024 11:59:37 +0200 Subject: [PATCH 1/6] Fix whitespace chars broke Dracut config parsing With commit 0785531e40c20404d24f1511b37a797b4fca3d7f the `get_config` function in anaconda-lib.sh was broken because missing quotes removed leading and trailing whitespace characters automatically but after the fix in commit mentioned above this side effect was fixed which lead in broken code. In other words the key were never matched because of trailing whitespace. Issue raised by this is not being able to read .treeinfo and .buildstamp files in Dracut. Example of such situation is broken boot when stage2 image is stored under special path mentioned in .treeinfo file. (cherry picked from commit 2336ba8bb7b449a86f698dedb7b9417043c9d94e) Related: RHEL-48821 --- dracut/anaconda-lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dracut/anaconda-lib.sh b/dracut/anaconda-lib.sh index e91e7f65137..8f3e41fab7f 100755 --- a/dracut/anaconda-lib.sh +++ b/dracut/anaconda-lib.sh @@ -26,6 +26,10 @@ config_get() { \[*\]*) cursec="${line#[}"; cursec="${cursec%%]*}" ;; *=*) k="${line%%=*}"; v="${line#*=}" ;; esac + # trim leading and trailing whitespace characters + k=$(echo "$k" | sed 's/^\s*//;s/\s*$//') + v=$(echo "$v" | sed 's/^\s*//;s/\s*$//') + if [ "$cursec" = "$section" ] && [ "$k" == "$key" ]; then echo "$v" break From 03ceb4e8a5f0affdacc42a5225e93bb280627984 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 16 Jul 2024 13:26:14 +0200 Subject: [PATCH 2/6] Fix trailing `/` when downloading stage2 image In Dracut the URL concatenation from .treeinfo could also point you to directory above by `..`. However, if the `inst.repo=` argument was set with trailing `/` it will create URL which is not supported by curl. Supported: `os/../BaseOS/` Unsupported by curl: `os//../BaseOS/` Remove the trailing `/` to avoid this issue. (cherry picked from commit 5cb0300d9c287d6b62235bcf946ce94050aa2938) Resolves: RHEL-48821 --- dracut/anaconda-lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dracut/anaconda-lib.sh b/dracut/anaconda-lib.sh index 8f3e41fab7f..58ddafb0f64 100755 --- a/dracut/anaconda-lib.sh +++ b/dracut/anaconda-lib.sh @@ -112,6 +112,10 @@ anaconda_net_root() { local repo="$1" info "anaconda: fetching stage2 from $repo" + # Remove last `/` from repo to enable constructs like ...os/../BaseOS/image/install.img + # Otherwise curl will fail to work with `...os//../BaseOS...` + repo=${repo%/} + # Try to get the local path to stage2 from treeinfo. treeinfo=$(fetch_url "$repo/.treeinfo" 2> /tmp/treeinfo_err) && \ stage2=$(config_get stage2 mainimage < "$treeinfo") From c59f02b45fbc53611e0fa734cfc96784197594d0 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Sun, 19 Jan 2025 23:33:52 +0100 Subject: [PATCH 3/6] Add test for config_get dracut function This commit will also add code to enable testing of the Dracut shell scripts. (cherry picked from commit 69da1b4d457353dd077d5aacfdf2cd4357cdb581) Related: RHEL-48821 --- .../dracut_tests/test_dracut_anaconda-lib.py | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py diff --git a/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py b/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py new file mode 100644 index 00000000000..7b61f140122 --- /dev/null +++ b/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py @@ -0,0 +1,167 @@ +# +# Copyright 2025 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2. This program is distributed in the hope that it +# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the +# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat +# trademarks that are incorporated in the source code or documentation are not +# subject to the GNU General Public License and may only be used or replicated +# with the express permission of Red Hat, Inc. + +import os +import re +import subprocess +import unittest + +from collections import namedtuple +from tempfile import NamedTemporaryFile, TemporaryDirectory + +DISABLE_COMMANDS = [ + "command", +] +DISABLE_COMMAND_PREFIX = "disabled-command" + + +SubprocessReturn = namedtuple("SubprocessReturn", + ["returncode", "disabled_cmd_args", "stdout", "stderr"]) + + +class AnacondaLibTestCase(unittest.TestCase): + + def setUp(self): + self._temp_dir = TemporaryDirectory() + self._content = "" + + def tearDown(self): + self._temp_dir.cleanup() + + def _load_script(self, script_name): + with open(os.path.join("../dracut/", script_name), "rt", encoding="utf-8") as f: + self._content = f.read() + + self._disable_bash_commands() + + def _disable_bash_commands(self): + disable_list = [] + # disable external and problematic commands in Dracut + for disabled_cmd in DISABLE_COMMANDS: + disable_list.append(f""" +{disabled_cmd}() {{ + echo "{DISABLE_COMMAND_PREFIX}: {disabled_cmd} args: $@" >&2 +}} +""") + + lines = self._content.splitlines() + self._content = lines[0] + "\n" + "\n".join(disable_list) + "\n" + "\n".join(lines[1:]) + + def _run_shell_command(self, command): + """Run a shell command and return the output + + This function will also split out disabled commands args from the stdout and returns + it as named tuple. + + :returns: SubprocessReturn named tuple + """ + command = f"{self._content}\n\n{command}" + ret = subprocess.run( + ["bash", "-c", command], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + check=False, + ) + + disabled_cmd_args, stderr = self._separate_disabled_commands_msgs(ret.stderr) + + return SubprocessReturn( + returncode=ret.returncode, + disabled_cmd_args=disabled_cmd_args, + stdout=ret.stdout.strip(), + stderr=stderr + ) + + def _separate_disabled_commands_msgs(self, stderr): + stderr_final = "" + disabled_cmd_args = {} + for line in stderr.splitlines(): + if line.startswith(DISABLE_COMMAND_PREFIX): + match = re.search(fr"{DISABLE_COMMAND_PREFIX}: ([\w-]+) args: (.*)$", line) + if match.group(1) in disabled_cmd_args: + disabled_cmd_args[match.group(1)].append(match.group(2)) + else: + disabled_cmd_args[match.group(1)] = [match.group(2)] + continue + + stderr_final += line + "\n" + + return disabled_cmd_args, stderr_final + + def _check_get_text_with_content(self, test_input, expected_stdout): + with NamedTemporaryFile(mode="wt", delete_on_close=False) as test_file: + test_file.write(test_input) + test_file.close() + ret = self._run_shell_command(f"config_get tree arch < {test_file.name}") + assert ret.returncode == 0 + assert ret.stdout == expected_stdout + + def test_config_get(self): + """Test bash config_get function to read .treeinfo file""" + self._load_script("anaconda-lib.sh") + + # test multiple values in file + self._check_get_text_with_content( + """ +[tree] +arch=x86_64 +[config] +abc=cde +""", + "x86_64", + ) + + # test space before and after '=' + self._check_get_text_with_content( + """ +[tree] +arch = aarch64 +[config] +abc=cde +""", + "aarch64", + ) + + # test multiple spaces before and after '=' + self._check_get_text_with_content( + """ +[tree] +arch =\t ppc64 +[config] +abc\t=\t\tcde +""", + "ppc64", + ) + + # test indented section + self._check_get_text_with_content( + """ + [tree] +\tarch = ppc64le +""", + "ppc64le", + ) + + # test indented value in section + self._check_get_text_with_content( + """ + [tree] + arch = s390 +""", + "s390", + ) From c19c5026e2d395c09ed3c55d62d9b6ee428695bd Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Mon, 20 Jan 2025 17:27:29 +0100 Subject: [PATCH 4/6] Enable dynamic bash commands disable for tests Dracut testing should be more versatile with this approach. This is more an extension for future testing. (cherry picked from commit 29debf114196dcb2c65c2c672b58b653298422f3) Related: RHEL-48821 --- .../dracut_tests/test_dracut_anaconda-lib.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py b/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py index 7b61f140122..984983e7853 100644 --- a/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py +++ b/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py @@ -19,13 +19,9 @@ import re import subprocess import unittest - from collections import namedtuple from tempfile import NamedTemporaryFile, TemporaryDirectory -DISABLE_COMMANDS = [ - "command", -] DISABLE_COMMAND_PREFIX = "disabled-command" @@ -46,13 +42,19 @@ def _load_script(self, script_name): with open(os.path.join("../dracut/", script_name), "rt", encoding="utf-8") as f: self._content = f.read() - self._disable_bash_commands() - - def _disable_bash_commands(self): + def _disable_bash_commands(self, disabled_commands): disable_list = [] # disable external and problematic commands in Dracut - for disabled_cmd in DISABLE_COMMANDS: - disable_list.append(f""" + for disabled_cmd in disabled_commands: + if isinstance(disabled_cmd, list): + disable_list.append(f""" +{disabled_cmd[0]}() {{ + echo "{DISABLE_COMMAND_PREFIX}: {disabled_cmd} args: $@" >&2 + {disabled_cmd[1]} +}} +""") + if isinstance(disabled_cmd, str): + disable_list.append(f""" {disabled_cmd}() {{ echo "{DISABLE_COMMAND_PREFIX}: {disabled_cmd} args: $@" >&2 }} @@ -114,6 +116,7 @@ def _check_get_text_with_content(self, test_input, expected_stdout): def test_config_get(self): """Test bash config_get function to read .treeinfo file""" self._load_script("anaconda-lib.sh") + self._disable_bash_commands(["command"]) # test multiple values in file self._check_get_text_with_content( From fd89d012b840962f9a0cd1e8de23d4f95d29a5e9 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Thu, 23 Jan 2025 16:12:16 +0100 Subject: [PATCH 5/6] Move shell testing to a separated directory For better visibility that we have such a tests. (cherry picked from commit 7e9ed4e09c14e801cdc491d160acb2877fcbf00a) Related: RHEL-48821 --- tests/unit_tests/shell_tests/__init__.py | 0 .../{dracut_tests => shell_tests}/test_dracut_anaconda-lib.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit_tests/shell_tests/__init__.py rename tests/unit_tests/{dracut_tests => shell_tests}/test_dracut_anaconda-lib.py (100%) diff --git a/tests/unit_tests/shell_tests/__init__.py b/tests/unit_tests/shell_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py b/tests/unit_tests/shell_tests/test_dracut_anaconda-lib.py similarity index 100% rename from tests/unit_tests/dracut_tests/test_dracut_anaconda-lib.py rename to tests/unit_tests/shell_tests/test_dracut_anaconda-lib.py From 912a9223a5cf115b6724e925e74eb886726dbcde Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Fri, 24 Jan 2025 16:07:27 +0100 Subject: [PATCH 6/6] Update the tests/README.rst file - Remove outdated parts - Fix minor issues - Add missing parts - Improve security topics Related: RHEL-48821 --- tests/README.rst | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/README.rst b/tests/README.rst index d6a1192186f..052b3de3e79 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -121,7 +121,7 @@ Run unit tests with patched pykickstart or other libraries 2. Run the container temporary with your required resources (pykickstart in this example):: - podman run --name=cnt-add --rm -it -v pykickstart:/pykickstart:z quay.io/rhinstaller/anaconda-ci:main sh + podman run --name=cnt-add --rm -it -v ./pykickstart:/pykickstart:z quay.io/rhinstaller/anaconda-ci:main sh 3. Do your required changes in the container (install pykickstart in this example):: @@ -177,16 +177,11 @@ ________________________ The `kickstart-tests.yml workflow`_ allows rhinstaller organization members to run kickstart-tests_ against an anaconda PR (only ``main`` for now). Send a -comment that starts with ``/kickstart-tests `` to the pull -request to trigger this. See the `kickstart launch script`_ documentation and -its ``--help`` for details what is supported; the two basic modes are running -a set of individual tests:: - - /kickstart-tests keyboard [test2 test3 ...] - -or running all tests of one or more given types:: - - /kickstart-tests --testtype network,autopart +comment that starts with ``/kickstart-tests `` to the pull request to +trigger it. It is possible to use tests updated via a kickstart-tests +repository PR. See the `kickstart-tests.yml workflow`_ for supported +options. For more detailed information on tests selection see the +`kickstart launch script`_ documentation and-its ``--help`` Container maintenance --------------------- @@ -277,7 +272,7 @@ represents a different class of tests. They are - *cppcheck/* - static C/C++ code analysis using the *cppcheck* tool; - *shellcheck/* - shell code analyzer config; -- *dd_tests/* - Python unit tests for driver disk utilities (utils/dd); +- *dd_tests/* - Python unit tests for driver disk utilities (dracut/dd); - *unit_tests/dracut_tests/* - Python unit tests for the dracut hooks used to configure the installation environment and load Anaconda; - *gettext/* - sanity tests of files used for translation; Written in Python and