Apply plugin envvars to phases not on command line

docs/overview.rst

@@ -734,7 +734,7 @@ belongs to, and the option name:
 
     ``TMT_PLUGIN_${STEP}_${PLUGIN}_${OPTION}``
 
-All values are upper-cased, with dashes (``-``) replaced by
+All names are upper-cased, with dashes (``-``) replaced by
 underscores (``_``).
 
 For example, an execute plugin "tmt" would run with verbosity
@@ -762,29 +762,11 @@ The following commands would override the URL::
 
     TMT_PLUGIN_DISCOVER_FMF_URL=https://actual.org/ tmt run ...
 
-For setting flag-like option, 0 and 1 are the expected value. For
-example, an interactive mode would be enabled in this run::
+For setting flag-like option, ``0`` and ``1`` are the expected values.
+For example, an interactive mode would be enabled in this run::
 
     TMT_PLUGIN_EXECUTE_TMT_INTERACTIVE=1 tmt run ... execute -h tmt ...
 
-.. note::
-
-   The following applies to situations when a plugin is specified
-   on the command line only. Keys of plugins specified in fmf files
-   would not be modified. This is a limit of the current implementation,
-   and will be addressed in the future::
-
-      # Here the verbosity will not be increased since the plugin is
-      # not mentioned on the command line:
-      $ TMT_PLUGIN_DISCOVER_FMF_VERBOSE=2 tmt run -a
-
-      # Here the environment variable will take effect:
-      $ TMT_PLUGIN_DISCOVER_FMF_VERBOSE=2 tmt run -a discover -h fmf ...
-
-   Several plugins (``report -h reportportal``, ``report -h polarion``,
-   ``execute -h tmt``) allow selected variables to be processed,
-   even when plugin is not specified on the command line.
-
 .. _regular-expressions:
 
 Regular Expressions

tests/cli/plugin-envvars/data/.fmf/version

@@ -0,0 +1 @@
+1

tests/cli/plugin-envvars/data/main.fmf

@@ -0,0 +1,10 @@
+provision:
+  how: local
+
+execute:
+  script: /bin/true
+
+report:
+  - how: html
+    absolute-paths: false
+    display-guest: auto

tests/cli/plugin-envvars/main.fmf

@@ -0,0 +1,4 @@
+summary: Verify some basic output of `tmt about`
+
+tag+:
+  - provision-local

tests/cli/plugin-envvars/test.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+rlJournalStart
+    rlPhaseStartSetup
+        rlRun "set -o pipefail"
+        rlRun "export LANG=C"
+        rlRun "rundir=$(mktemp -d)"
+
+        rlRun "pushd data"
+    rlPhaseEnd
+
+    rlPhaseStartTest "Foo"
+        rlRun "run_tmt=\"tmt -vvv --feeling-safe --log-topic=cli-invocations run --id $rundir --scratch\""
+
+        rlRun -s "$run_tmt"
+        rlAssertGrep "ReportPlugin.delegate\(step=report, data=None, raw_data=\{'how': 'html', 'absolute-paths': False, 'display-guest': 'auto', 'name': 'default-0'\}\)" $rundir/log.txt -E
+
+        rlRun -s "TMT_PLUGIN_REPORT_HTML_ABSOLUTE_PATHS=1 TMT_PLUGIN_REPORT_HTML_FILE=/tmp/foo TMT_PLUGIN_REPORT_HTML_DISPLAY_GUEST=never $run_tmt"
+        rlAssertGrep "ReportPlugin.delegate\(step=report, data=None, raw_data=\{'how': 'html', 'absolute-paths': True, 'display-guest': 'never', 'name': 'default-0', 'file': '/tmp/foo'\}\)" $rundir/log.txt -E
+    rlPhaseEnd
+
+    rlPhaseStartTest "Verify unknown plugins are reported"
+        rlRun -s "TMT_PLUGIN_REPORT_XHTML_DISPLAY_GUEST=never $run_tmt"
+
+        rlAssertGrep "warn: Found environment variables for plugin 'report/xhtml', but the plugin was not found. The following environment variables will have no effect:" $rlRun_LOG
+        rlAssertGrep "warn: TMT_PLUGIN_REPORT_XHTML_DISPLAY_GUEST" $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartTest "Verify unused envvars are reported"
+        rlRun -s "TMT_PLUGIN_REPORT_DISPLAY_DISPLAY_GUEST=never $run_tmt"
+
+        rlAssertGrep "warn: Found environment variables for plugin 'report/display', but the plugin is not used by the plan '/'. The following environment variables will have no effect:" $rlRun_LOG
+        rlAssertGrep "warn: TMT_PLUGIN_REPORT_DISPLAY_DISPLAY_GUEST" $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartTest "Verify unknown options are reported"
+        rlRun -s "TMT_PLUGIN_REPORT_HTML_HIDE_GUEST=never $run_tmt" 2
+
+        rlAssertGrep "Failed to find the 'hide-guest' key of the 'report/html' plugin." $rlRun_LOG
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        rlRun "popd"
+        rlRun "rm -rf $rundir"
+    rlPhaseEnd
+rlJournalEnd

tmt/steps/__init__.py

@@ -61,6 +61,7 @@
     Command,
     CommandOutput,
     Environment,
+    EnvVarName,
     EnvVarValue,
     GeneralError,
     HasPhaseWorkdir,
@@ -132,6 +133,11 @@
 
 CODE_BLOCK_REGEXP = re.compile(r"^\s*\.\. code-block::.*$\n", re.MULTILINE)
 
+#: A regular expression to cut the special, plugin-specific environment
+#: variable names into their components: step, plugin, and option names.
+PLUGIN_ENVIRONMENT_VARIABLE_NAME_PATTERN = re.compile(
+    r'TMT_PLUGIN_(?P<step>[A-Z]+)_(?P<plugin>[A-Z]+)_(?P<option>[A-Z_]+)'
+)
 
 PHASE_OPTIONS = tmt.options.create_options_decorator(
     [
@@ -1026,6 +1032,35 @@ def _iter_options() -> Iterator[tuple[str, Any]]:
         # and we will get back to them once we're done with those we can apply immediately.
         postponed_invocations: list[tmt.cli.CliInvocation] = []
 
+        # A container of all plugin-specific environment variables. Here we collect all
+        # plugin-specific environment variables, and separate them into buckets per plugin. Note
+        # that we do not care about *phases* - there is no way to apply environment variable to
+        # one specific phase, they are very strong and affect all phases spawned from the same
+        # plugin, i.e. with the same `how`.
+        environment_invocations: dict[str, dict[str, tuple[EnvVarName, EnvVarValue]]] = (
+            collections.defaultdict(dict)
+        )
+
+        for envvar_name, envvar_value in Environment.from_environ().items():
+            # Skip all environment variables not matching the special pattern...
+            match = PLUGIN_ENVIRONMENT_VARIABLE_NAME_PATTERN.match(envvar_name)
+
+            if not match:
+                continue
+
+            # ... or for different step. Neither of these groups is interesting here and now.
+            step_name = match.group(1).lower()
+
+            if step_name != self.step_name:
+                continue
+
+            # Since the environment variable names are upper-cased only, plugin and field names
+            # require a bit of character manipulation.
+            plugin_name: str = match.group(2).lower().replace('_', '-')
+            option_name: str = key_to_option(match.group(3).lower())
+
+            environment_invocations[plugin_name][option_name] = (envvar_name, envvar_value)
+
         name_generator = DefaultNameGenerator.from_raw_phases(raw_data)
 
         def _ensure_name(raw_datum: _RawStepData) -> _RawStepData:
@@ -1096,18 +1131,96 @@ def _patch_raw_datum(
             self.plan._applied_cli_invocations.append(invocation)
 
         # A bit of logging before we start messing with step data
-        for i, raw_datum in enumerate(raw_data):
-            debug2(f'raw step datum #{i}', str(raw_datum))
+        def _log_raw_data(stage: str, raw_data: list[_RawStepData]) -> None:
+            debug2(f'stage {stage}')
+
+            for i, raw_datum in enumerate(raw_data):
+                debug3(f'raw step datum #{i}', str(raw_datum))
+
+        how: Optional[str]
+
+        _log_raw_data('before', raw_data)
+
+        # First pass, apply environment variables
+        for i, (how, plugin_environment) in enumerate(environment_invocations.items()):
+            debug2(f'environment invocation #{i}', f'{how}: {plugin_environment}')
+
+            original_environment_variable_names = [
+                envvar_name for (envvar_name, _) in plugin_environment.values()
+            ]
+
+            # We do not want to invent our custom parsers. Instead, we need to reach the Click
+            # command of our plugin, because that command already knows how to handle its input.
+            # This requires a bit of traversal. Get the right plugin class, then its command, and
+            # then its parameters.
+            plugin_class = self._plugin_base_class._supported_methods.get_plugin(how)
+
+            if plugin_class is None:
+                self.warn(
+                    f"Found environment variables for plugin '{self.step_name}/{how}',"
+                    " but the plugin was not found. The following environment variables"
+                    " will have no effect:"
+                )
+
+                self.warn(fmf.utils.listed(original_environment_variable_names))
+
+                continue
+
+            plugin_command = plugin_class.class_.command()
+            plugin_context = plugin_command.make_context(info_name=None, args=[])
+
+            # Apparently, Click does not offer a mapping between option names and their
+            # `click.Parameter` descriptions. We shall build our own then, we will need it.
+            plugin_params = {param.name: param for param in plugin_command.params}
+
+            # Now, traverse all raw data, skip those for different plugins, and update the suitable
+            # ones with values coming from the environment variables.
+            for j, raw_datum in enumerate(raw_data):
+                debug3(f'raw step datum #{j}', str(raw_datum))
+
+                compatible_raw_data = [
+                    raw_datum for raw_datum in raw_data if raw_datum.get('how') == how
+                ]
+
+                if not compatible_raw_data:
+                    self.warn(
+                        f"Found environment variables for plugin '{self.step_name}/{how}',"
+                        f" but the plugin is not used by the plan '{self.plan.name}'. The"
+                        " following environment variables will have no effect:"
+                    )
+
+                    self.warn(fmf.utils.listed(original_environment_variable_names))
+
+                    continue
 
-        # The first pass, apply CLI invocations that can be applied
+                for option_name, (envvar_name, envvar_value) in plugin_environment.items():
+                    debug4('raw environment variable', f'{envvar_name}={envvar_value}')
+
+                    param_name = option_to_key(option_name)
+
+                    plugin_param = plugin_params.get(param_name)
+
+                    if plugin_param is None:
+                        raise GeneralError(
+                            f"Failed to find the '{param_name.replace('_', '-')}' key"
+                            f" of the '{self.step_name}/{how}' plugin."
+                        )
+
+                    raw_datum[key_to_option(param_name)] = plugin_param.type_cast_value(  # type: ignore[literal-required]
+                        plugin_context, str(envvar_value)
+                    )
+
+        _log_raw_data('after environment invocations', raw_data)
+
+        # Second pass, apply CLI invocations that can be applied
         for i, invocation in enumerate(self.__class__.cli_invocations):
             debug2(f'invocation #{i}', str(invocation.options))
 
             if invocation in self.plan._applied_cli_invocations:
                 debug3('already applied')
                 continue
 
-            how: Optional[str] = invocation.options.get('how')
+            how = cast(Optional[str], invocation.options.get('how'))
 
             if how is None:
                 debug3('how-less phase (postponed)')
@@ -1183,7 +1296,9 @@ def _patch_raw_datum(
 
                 postponed_invocations.append(invocation)
 
-        # The second pass, evaluate postponed CLI invocations
+        _log_raw_data('after targeted CLI invocations', raw_data)
+
+        # Third pass, evaluate postponed CLI invocations
         for i, invocation in enumerate(postponed_invocations):
             debug2(f'postponed invocation #{i}', str(invocation.options))
 
@@ -1194,7 +1309,7 @@ def _patch_raw_datum(
             # preferred image name without specifying the provision
             # method, thus the 'how' key can be unset and we respect the
             # provision method specified in the plan.
-            how = invocation.options.get('how')
+            how = cast(Optional[str], invocation.options.get('how'))
 
             for j, raw_datum in enumerate(raw_data):
                 debug2(f'raw step datum #{j}', str(raw_datum))
@@ -1244,8 +1359,7 @@ def _patch_raw_datum(
             raw_data = pruned_raw_data
 
         # And bit of logging after re're done with CLI invocations
-        for i, raw_datum in enumerate(raw_data):
-            debug2(f'updated raw step datum #{i}', str(raw_datum))
+        _log_raw_data('after general CLI invocations', raw_data)
 
         raw_data = self._set_default_names(raw_data)
         return self._set_default_how(raw_data)