diff --git a/dissect/target/helpers/docs.py b/dissect/target/helpers/docs.py index c662918cd..522e96a1b 100644 --- a/dissect/target/helpers/docs.py +++ b/dissect/target/helpers/docs.py @@ -1,7 +1,9 @@ import inspect import itertools import textwrap -from typing import Any, Callable, Tuple, Type +from typing import Any, Callable + +from dissect.target.plugin import Plugin NO_DOCS = "No documentation" @@ -14,76 +16,23 @@ INDENT_STEP = " " * 4 - -def get_plugin_class_for_func(func: Callable) -> Type: - """Return pluging class for provided function instance""" - func_parent_name = func.__qualname__.rsplit(".", 1)[0] - klass = getattr(inspect.getmodule(func), func_parent_name, None) - return klass - - -def get_real_func_obj(func: Callable) -> Tuple[Type, Callable]: - """Return a tuple with plugin class and underlying func object for provided function instance""" - klass = None - - if isinstance(func, property): - # turn property into function - func = func.fget - - if inspect.ismethod(func): - for klass in inspect.getmro(func.__self__.__class__): - if func.__name__ in klass.__dict__: - break - else: - func = getattr(func, "__func__", func) - - if inspect.isfunction(func): - klass = get_plugin_class_for_func(func) - - if not klass: - raise ValueError(f"Can't find class for {func}") - - return (klass, func) +FUNC_DOC_TEMPLATE = "{func_name} - {short_description} (output: {output_type})" def get_docstring(obj: Any, placeholder: str = NO_DOCS) -> str: - """Get object's docstring or a placeholder if no docstring found""" + """Get object's docstring or a placeholder if no docstring found.""" # Use of `inspect.cleandoc()` is preferred to `textwrap.dedent()` here # because many multi-line docstrings in the codebase # have no indentation in the first line, which confuses `dedent()` return inspect.cleandoc(obj.__doc__) if obj.__doc__ else placeholder -def get_func_details(func: Callable) -> Tuple[str, str]: - """Return a tuple with function's name, output label and docstring""" - func_doc = get_docstring(func) - - if hasattr(func, "__output__") and func.__output__ in FUNCTION_OUTPUT_DESCRIPTION: - func_output = FUNCTION_OUTPUT_DESCRIPTION[func.__output__] - else: - func_output = "unknown" - - return (func_output, func_doc) - - -def get_full_func_name(plugin_class: Type, func: Callable) -> str: - func_name = func.__name__ - - if hasattr(plugin_class, "__namespace__") and plugin_class.__namespace__: - func_name = f"{plugin_class.__namespace__}.{func_name}" - - return func_name - - -FUNC_DOC_TEMPLATE = "{func_name} - {short_description} (output: {output_type})" - - def get_func_description(func: Callable, with_docstrings: bool = False) -> str: - klass, func = get_real_func_obj(func) - func_output, func_doc = get_func_details(func) + klass, func = _get_real_func_obj(func) + func_output, func_doc = _get_func_details(func) # get user-friendly function name - func_name = get_full_func_name(klass, func) + func_name = _get_full_func_name(klass, func) if with_docstrings: func_title = f"`{func_name}` (output: {func_output})" @@ -98,15 +47,15 @@ def get_func_description(func: Callable, with_docstrings: bool = False) -> str: return desc -def get_plugin_functions_desc(plugin_class: Type, with_docstrings: bool = False) -> str: +def get_plugin_functions_desc(plugin_class: type[Plugin], with_docstrings: bool = False) -> str: descriptions = [] for func_name in plugin_class.__exports__: func_obj = getattr(plugin_class, func_name) - if getattr(func_obj, "get_func_doc_spec", None): - func_desc = FUNC_DOC_TEMPLATE.format_map(func_obj.get_func_doc_spec()) - else: - _, func = get_real_func_obj(func_obj) - func_desc = get_func_description(func, with_docstrings=with_docstrings) + if func_obj is getattr(plugin_class, "__call__", None): + continue + + _, func = _get_real_func_obj(func_obj) + func_desc = get_func_description(func, with_docstrings=with_docstrings) descriptions.append(func_desc) # sort functions in the plugin alphabetically @@ -120,7 +69,7 @@ def get_plugin_functions_desc(plugin_class: Type, with_docstrings: bool = False) return paragraph -def get_plugin_description(plugin_class: Type) -> str: +def get_plugin_description(plugin_class: type[Plugin]) -> str: plugin_name = plugin_class.__name__ plugin_desc_title = f"`{plugin_name}` (`{plugin_class.__module__}.{plugin_name}`)" plugin_doc = textwrap.indent(get_docstring(plugin_class), prefix=INDENT_STEP) @@ -128,7 +77,9 @@ def get_plugin_description(plugin_class: Type) -> str: return paragraph -def get_plugin_overview(plugin_class: Type, with_plugin_desc: bool = False, with_func_docstrings: bool = False) -> str: +def get_plugin_overview( + plugin_class: type[Plugin], with_plugin_desc: bool = False, with_func_docstrings: bool = False +) -> str: paragraphs = [] if with_plugin_desc: @@ -150,3 +101,55 @@ def get_plugin_overview(plugin_class: Type, with_plugin_desc: bool = False, with paragraphs.append(func_descriptions_paragraph) overview = "\n".join(paragraphs) return overview + + +def _get_plugin_class_for_func(func: Callable) -> type[Plugin]: + """Return plugin class for provided function instance.""" + func_parent_name = func.__qualname__.rsplit(".", 1)[0] + klass = getattr(inspect.getmodule(func), func_parent_name, None) + return klass + + +def _get_real_func_obj(func: Callable) -> tuple[type[Plugin], Callable]: + """Return a tuple with plugin class and underlying function object for provided function instance.""" + klass = None + + if isinstance(func, property): + # turn property into function + func = func.fget + + if inspect.ismethod(func): + for klass in inspect.getmro(func.__self__.__class__): + if func.__name__ in klass.__dict__: + break + else: + func = getattr(func, "__func__", func) + + if inspect.isfunction(func): + klass = _get_plugin_class_for_func(func) + + if not klass: + raise ValueError(f"Can't find class for {func}") + + return (klass, func) + + +def _get_func_details(func: Callable) -> tuple[str, str]: + """Return a tuple with function's name, output label and docstring""" + func_doc = get_docstring(func) + + if hasattr(func, "__output__") and func.__output__ in FUNCTION_OUTPUT_DESCRIPTION: + func_output = FUNCTION_OUTPUT_DESCRIPTION[func.__output__] + else: + func_output = "unknown" + + return (func_output, func_doc) + + +def _get_full_func_name(plugin_class: type[Plugin], func: Callable) -> str: + func_name = func.__name__ + + if hasattr(plugin_class, "__namespace__") and plugin_class.__namespace__: + func_name = f"{plugin_class.__namespace__}.{func_name}" + + return func_name diff --git a/dissect/target/plugin.py b/dissect/target/plugin.py index 7b4fcaab5..4100f31d7 100644 --- a/dissect/target/plugin.py +++ b/dissect/target/plugin.py @@ -9,42 +9,42 @@ import functools import importlib import importlib.util -import inspect import logging import os import sys import traceback -from collections import defaultdict from dataclasses import dataclass, field +from itertools import zip_longest from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Type +from typing import TYPE_CHECKING, Any, Callable, Iterator + +try: + from typing import TypeAlias # novermin +except ImportError: + # COMPAT: Remove this when we drop Python 3.9 + TypeAlias = Any from flow.record import Record, RecordDescriptor -import dissect.target.plugins.general as general +import dissect.target.plugins.os.default as default from dissect.target.exceptions import PluginError, UnsupportedPluginError from dissect.target.helpers import cache +from dissect.target.helpers.fsutil import has_glob_magic from dissect.target.helpers.record import EmptyRecord from dissect.target.helpers.utils import StrEnum -try: - from dissect.target.plugins._pluginlist import PLUGINS - - GENERATED = True -except Exception: - PLUGINS = {} - GENERATED = False - if TYPE_CHECKING: from dissect.target import Target from dissect.target.filesystem import Filesystem from dissect.target.helpers.record import ChildTargetRecord -PluginDescriptor = dict[str, Any] -"""A dictionary type, for what the plugin descriptor looks like.""" +log = logging.getLogger(__name__) MODULE_PATH = "dissect.target.plugins" """The base module path to the in-tree plugins.""" + +OS_MODULE_PATH = "dissect.target.plugins.os" + OUTPUTS = ( "default", "record", @@ -53,8 +53,6 @@ ) """The different output types supported by ``@export``.""" -log = logging.getLogger(__name__) - class OperatingSystem(StrEnum): ANDROID = "android" @@ -71,7 +69,117 @@ class OperatingSystem(StrEnum): WINDOWS = "windows" -def export(*args, **kwargs) -> Callable: +@dataclass(frozen=True, eq=True) +class PluginDescriptor: + # COMPAT: Replace with slots=True when we drop Python 3.9 + __slots__ = ("module", "qualname", "namespace", "path", "findable", "functions", "exports") + + module: str + qualname: str + namespace: str + path: str + findable: bool + functions: list[str] + exports: list[str] + + +@dataclass(frozen=True, eq=True) +class FunctionDescriptor: + # COMPAT: Replace with slots=True when we drop Python 3.9 + __slots__ = ( + "name", + "namespace", + "path", + "exported", + "internal", + "findable", + "output", + "method_name", + "module", + "qualname", + ) + + name: str + namespace: str + path: str + exported: bool + internal: bool + findable: bool + output: str | None + method_name: str + module: str + qualname: str + + +@dataclass(frozen=True, eq=True) +class FailureDescriptor: + # COMPAT: Replace with slots=True when we drop Python 3.9 + __slots__ = ("module", "stacktrace") + + module: str + stacktrace: list[str] + + +# COMPAT: Add slots=True when we drop Python 3.9 +# We can't manually define __slots__ here because we use have to use field() for the default_factory +@dataclass(frozen=True) +class PluginDescriptorLookup: + # All regular plugins + # {"": PluginDescriptor} + __regular__: dict[str, PluginDescriptor] = field(default_factory=dict) + # All OS plugins + # {"": PluginDescriptor} + __os__: dict[str, PluginDescriptor] = field(default_factory=dict) + # All child plugins + # {"": PluginDescriptor} + __child__: dict[str, PluginDescriptor] = field(default_factory=dict) + + +# COMPAT: Add slots=True when we drop Python 3.9 +# We can't manually define __slots__ here because we use have to use field() for the default_factory +@dataclass(frozen=True) +class FunctionDescriptorLookup: + # All regular plugins + # {"": {"": FunctionDescriptor}} + __regular__: dict[str, dict[str, FunctionDescriptor]] = field(default_factory=dict) + # All OS plugins + # {"": {"": FunctionDescriptor}} + __os__: dict[str, dict[str, FunctionDescriptor]] = field(default_factory=dict) + # All child plugins + # {"": {"": FunctionDescriptor}} + __child__: dict[str, dict[str, FunctionDescriptor]] = field(default_factory=dict) + + +_OSTree: TypeAlias = dict[str, "_OSTree"] + + +# COMPAT: Add slots=True when we drop Python 3.9 +# We can't manually define __slots__ here because we use have to use field() for the default_factory +@dataclass(frozen=True) +class PluginRegistry: + # Plugin descriptor lookup + __plugins__: PluginDescriptorLookup = field(default_factory=PluginDescriptorLookup) + # Function descriptor lookup + __functions__: FunctionDescriptorLookup = field(default_factory=FunctionDescriptorLookup) + # OS plugin tree + # {"": {"": {}}} + __ostree__: _OSTree = field(default_factory=dict) + # Failures + __failed__: list[FailureDescriptor] = field(default_factory=list) + + +PLUGINS: PluginRegistry = PluginRegistry() +"""The plugin registry. + +Note: It's very important that all values in this dictionary are serializable. +The plugin registry can be stored in a file and loaded later. Plain Python syntax is used to store the registry. +An exception is made for :class:`FailureDescriptor`, :class:`FunctionDescriptor` and :class:`PluginDescriptor`. +""" + +GENERATED = False + + +def export(*args, **kwargs) -> Callable[..., Any]: """Decorator to be used on Plugin functions that should be exported. Supported keyword arguments: @@ -81,7 +189,8 @@ def export(*args, **kwargs) -> Callable: cache (bool): Whether the result of this function should be cached. record (RecordDescriptor): The :class:`flow.record.RecordDescriptor` for the records that this function yields. - If the records are dynamically made, use DynamicRecord instead. + If multiple record types are yielded, specificy each descriptor in a list. + If the records are dynamically made, use :func:`dissect.target.helpers.record.DynamicDescriptor` instead. output (str): The output type of this function. Must be one of: - default: Single return value @@ -101,10 +210,10 @@ def export(*args, **kwargs) -> Callable: An exported function from a plugin. """ - def decorator(obj): + def decorator(obj: Callable[..., Any]) -> Callable[..., Any] | property: # Properties are implicitly cached # Important! Currently it's crucial that this is *always* called - # See the comment in Plugin.__init_subclass__ for more detail regarding Plugin.get_all_records + # See the comment in Plugin.__init_subclass__ for more detail regarding Plugin.__call__ obj = cache.wrap(obj, no_cache=not kwargs.get("cache", True), cls=kwargs.get("cls", None)) output = kwargs.get("output", "default") @@ -131,41 +240,90 @@ def decorator(obj): return decorator -def get_nonprivate_attribute_names(cls: Type[Plugin]) -> list[str]: - """Retrieve all attributes that do not start with ``_``.""" - return [attr for attr in dir(cls) if not attr.startswith("_")] +def internal(*args, **kwargs) -> Callable[..., Any]: + """Decorator to be used on plugin functions that should be internal only. + + Making a plugin internal means that it's only callable from the Python API and not through ``target-query``. + This decorator adds the ``__internal__`` private attribute to a method or property. + The attribute is always set to ``True``, to tell :func:`register` that it is an internal + method or property. + """ -def get_nonprivate_attributes(cls: Type[Plugin]) -> list[Any]: - """Retrieve all public attributes of a :class:`Plugin`.""" - # Note: `dir()` might return attributes from parent class - return [getattr(cls, attr) for attr in get_nonprivate_attribute_names(cls)] + def decorator(obj: Callable[..., Any]) -> Callable[..., Any] | property: + obj.__internal__ = True + if kwargs.get("property", False): + obj = property(obj) + return obj + if len(args) == 1: + return decorator(args[0]) + else: + return decorator -def get_nonprivate_methods(cls: Type[Plugin]) -> list[Callable]: - """Retrieve all public methods of a :class:`Plugin`.""" - return [attr for attr in get_nonprivate_attributes(cls) if not isinstance(attr, property) and callable(attr)] +def arg(*args, **kwargs) -> Callable[..., Any]: + """Decorator to be used on Plugin functions that accept additional command line arguments. -def get_descriptors_on_nonprivate_methods(cls: Type[Plugin]) -> list[RecordDescriptor]: - """Return record descriptors set on nonprivate methods in `cls` class.""" - descriptors = set() - methods = get_nonprivate_methods(cls) + Command line arguments can be added using the ``@arg`` decorator. + Arguments to this decorator are directly forwarded to the ``ArgumentParser.add_argument`` function of ``argparse``. + Resulting arguments are passed to the function using kwargs. + The keyword argument name must match the argparse argument name. - for m in methods: - if not hasattr(m, "__record__"): - continue + This decorator adds the ``__args__`` private attribute to a method or property. + This attribute holds all the command line arguments that were added to the plugin function. + """ - record = m.__record__ - if not record: - continue + def decorator(obj: Callable[..., Any]) -> Callable[..., Any]: + if not hasattr(obj, "__args__"): + obj.__args__ = [] - try: - # check if __record__ value is iterable (for example, a list) - descriptors.update(record) - except TypeError: - descriptors.add(record) - return list(descriptors) + obj.__args__.append((args, kwargs)) + + return obj + + return decorator + + +def alias(*args, **kwargs: dict[str, Any]) -> Callable[..., Any]: + """Decorator to be used on :class:`Plugin` functions to register an alias of that function.""" + + if not kwargs.get("name") and not args: + raise ValueError("Missing argument 'name'") + + def decorator(obj: Callable[..., Any]) -> Callable[..., Any]: + if not hasattr(obj, "__aliases__"): + obj.__aliases__ = [] + + if name := (kwargs.get("name") or args[0]): + obj.__aliases__.append(name) + + return obj + + return decorator + + +def clone_alias(cls: type, attr: Callable[..., Any], alias: str) -> None: + """Clone the given attribute to an alias in the provided class.""" + + # Clone the function object + clone = type(attr)(attr.__code__, attr.__globals__, alias, attr.__defaults__, attr.__closure__) + clone.__kwdefaults__ = attr.__kwdefaults__ + + # Copy some attributes + functools.update_wrapper(clone, attr) + if wrapped := getattr(attr, "__wrapped__", None): + # update_wrapper sets a new wrapper, we want the original + clone.__wrapped__ = wrapped + + # Update module path so we can fool inspect.getmodule with subclassed Plugin classes + clone.__module__ = cls.__module__ + + # Update the names + clone.__name__ = alias + clone.__qualname__ = f"{cls.__name__}.{alias}" + + setattr(cls, alias, clone) class Plugin: @@ -182,9 +340,8 @@ class attribute. Namespacing results in your plugin needing to be prefixed - ``__namespace__`` - ``__record_descriptors__`` - With the following three being assigned in :func:`register`: + With the following two being assigned in :func:`register`: - - ``__plugin__`` - ``__functions__`` - ``__exports__`` @@ -221,23 +378,25 @@ class attribute. Namespacing results in your plugin needing to be prefixed produce redundant results when used with a wild card (browser.* -> browser.history + browser.*.history). """ + __functions__: list[str] + """Internal. A list of all method names decorated with ``@internal`` or ``@export``.""" + __exports__: list[str] + """Internal. A list of all method names decorated with ``@export``.""" def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - # Do not register the "base" subclassess `OSPlugin` and `ChildTargetPlugin` - if cls.__name__ not in ("OSPlugin", "ChildTargetPlugin") and cls.__register__: + # Do not register the "base" subclasses defined in this file + if cls.__module__ != Plugin.__module__: register(cls) - record_descriptors = get_descriptors_on_nonprivate_methods(cls) + record_descriptors = _get_descriptors_on_nonprivate_methods(cls) cls.__record_descriptors__ = record_descriptors # This is a bit tricky currently - # cls.get_all_records is the *function* Plugin.get_all_records, not from the subclass + # cls.__call__ is the *function* Plugin.__call__, not from the subclass # export() currently will _always_ return a new object because it always calls ``cache.wrap(obj)`` - # This allows this to work, otherwise the Plugin.get_all_records would get all the plugin attributes set on it - cls.get_all_records = export(output="record", record=record_descriptors, cache=False, cls=cls)( - cls.get_all_records - ) + # This allows this to work, otherwise the Plugin.__call__ would get all the plugin attributes set on it + cls.__call__ = export(output="record", record=record_descriptors, cache=False, cls=cls)(cls.__call__) def __init__(self, target: Target): self.target = target @@ -264,607 +423,532 @@ def check_compatible(self) -> None: """ raise NotImplementedError - def get_all_records(self) -> Iterator[Record]: + def __call__(self, *args, **kwargs) -> Iterator[Record | Any]: """Return the records of all exported methods. Raises: PluginError: If the subclass is not a namespace plugin. """ if not self.__namespace__: - raise PluginError(f"Plugin {self.__class__.__name__} is not a namespace plugin") + raise PluginError(f"Plugin {self.__class__.__name__} is not a callable") for method_name in self.__exports__: + if method_name == "__call__": + continue + method = getattr(self, method_name) + if getattr(method, "__output__", None) not in ("record", "yield"): + continue try: yield from method() except Exception: self.target.log.error("Error while executing `%s.%s`", self.__namespace__, method_name, exc_info=True) - def __call__(self, *args, **kwargs): - """A shortcut to :func:`get_all_records`. - Raises: - PluginError: If the subclass is not a namespace plugin. - """ - if not self.__namespace__: - raise PluginError(f"Plugin {self.__class__.__name__} is not a callable") - return self.get_all_records() +def register(plugincls: type[Plugin]) -> None: + """Register a plugin, and put related data inside :attr:`PLUGINS`. + This function uses the following private attributes that are set using decorators: + - ``__exported__``: Set in :func:`export`. + - ``__internal__``: Set in :func:`internal`. -class OSPlugin(Plugin): - """Base class for OS plugins. + Additionally, ``register`` sets the following private attributes on the `plugincls`: + - ``__functions__``: A list of all the methods and properties that are ``__internal__`` or ``__exported__``. + - ``__exports__``: A list of all the methods or properties that were explicitly exported. - This provides a base class for certain common functions of OS's, which each OS plugin has to implement separately. + If a plugincls ``__register__`` attribute is set to ``False``, the plugin will not be registered, but the + plugin will still be processed for the private attributes mentioned above. - For example, it provides an interface for retrieving the hostname and users of a target. + Args: + plugincls: A plugin class to register. - All derived classes MUST implement ALL the classmethods and exported - methods with the same ``@classmethod`` or ``@export(...)`` annotation. + Raises: + ValueError: If ``plugincls`` is not a subclass of :class:`Plugin`. """ + if not issubclass(plugincls, Plugin): + raise ValueError("Not a subclass of Plugin") - def __init_subclass__(cls, **kwargs): - # Note that cls is the subclass - super().__init_subclass__(**kwargs) + # Register the plugin in the correct tree + __plugins__ = PLUGINS.__plugins__ + __functions__ = PLUGINS.__functions__ + if issubclass(plugincls, OSPlugin): + plugin_index = __plugins__.__os__ + function_index = __functions__.__os__ + elif issubclass(plugincls, ChildTargetPlugin): + plugin_index = __plugins__.__child__ + function_index = __functions__.__child__ + else: + plugin_index = __plugins__.__regular__ + function_index = __functions__.__regular__ - for os_method in get_nonprivate_attributes(OSPlugin): - if isinstance(os_method, property): - os_method = os_method.fget - os_docstring = os_method.__doc__ + exports = [] + functions = [] + module_path = _module_path(plugincls) + module_key = f"{module_path}.{plugincls.__qualname__}" - method = getattr(cls, os_method.__name__, None) - if isinstance(method, property): - method = method.fget - # This works as None has a __doc__ property (which is None). - docstring = method.__doc__ + if not issubclass(plugincls, ChildTargetPlugin): + # First pass to resolve aliases + for attr in _get_nonprivate_attributes(plugincls): + for alias in getattr(attr, "__aliases__", []): + clone_alias(plugincls, attr, alias) - if method and not docstring: - if hasattr(method, "__func__"): - method = method.__func__ - method.__doc__ = os_docstring + for attr in _get_nonprivate_attributes(plugincls): + if isinstance(attr, property): + attr = attr.fget - def check_compatible(self) -> bool: - """OSPlugin's use a different compatibility check, override the one from the :class:`Plugin` class. + if getattr(attr, "__generated__", False) and plugincls != plugincls.__nsplugin__: + continue - Returns: - This function always returns ``True``. - """ - return True + exported = getattr(attr, "__exported__", False) + internal = getattr(attr, "__internal__", False) - @classmethod - def detect(cls, fs: Filesystem) -> Optional[Filesystem]: - """Provide detection of this OSPlugin on a given filesystem. + if exported or internal: + functions.append(attr.__name__) + if exported: + exports.append(attr.__name__) - Args: - fs: :class:`~dissect.target.filesystem.Filesystem` to detect the OS on. + if plugincls.__register__: + name = attr.__name__ + if plugincls.__namespace__: + name = f"{plugincls.__namespace__}.{name}" - Returns: - The root filesystem / sysvol when found. - """ - raise NotImplementedError + path = f"{module_path}.{attr.__name__}" - @classmethod - def create(cls, target: Target, sysvol: Filesystem) -> OSPlugin: - """Initiate this OSPlugin with the given target and detected filesystem. + members = function_index.setdefault(name, {}) + if module_key in members: + continue - Args: - target: The :class:`~dissect.target.target.Target` object. - sysvol: The filesystem that was detected in the ``detect()`` function. + descriptor = FunctionDescriptor( + name=name, + namespace=plugincls.__namespace__, + path=path, + exported=exported, + internal=internal, + findable=plugincls.__findable__, + output=getattr(attr, "__output__", None), + method_name=attr.__name__, + module=plugincls.__module__, + qualname=plugincls.__qualname__, + ) - Returns: - An instantiated version of the OSPlugin. - """ - raise NotImplementedError + # Register the functions in the lookup + members[module_key] = descriptor + + if plugincls.__namespace__: + # Namespaces are also callable, so register the namespace itself as well + if module_key not in function_index.get(plugincls.__namespace__, {}): + functions.append("__call__") + if len(exports): + exports.append("__call__") + + if plugincls.__register__: + descriptor = FunctionDescriptor( + name=plugincls.__namespace__, + namespace=plugincls.__namespace__, + path=module_path, + exported=bool(len(exports)), + internal=bool(len(functions)) and not bool(len(exports)), + findable=plugincls.__findable__, + output=getattr(plugincls.__call__, "__output__", None), + method_name="__call__", + module=plugincls.__module__, + qualname=plugincls.__qualname__, + ) - @export(property=True) - def hostname(self) -> Optional[str]: - """Return the target's hostname. + function_index.setdefault(plugincls.__namespace__, {})[module_key] = descriptor - Returns: - The hostname as string. - """ - raise NotImplementedError + # Update the class with the plugin attributes + plugincls.__functions__ = functions + plugincls.__exports__ = exports - @export(property=True) - def ips(self) -> list[str]: - """Return the IP addresses configured in the target. + if plugincls.__register__: + if module_key in plugin_index: + return - Returns: - The IPs as list. - """ - raise NotImplementedError + plugin_index[module_key] = PluginDescriptor( + module=plugincls.__module__, + qualname=plugincls.__qualname__, + namespace=plugincls.__namespace__, + path=module_path, + findable=plugincls.__findable__, + functions=functions, + exports=exports, + ) - @export(property=True) - def version(self) -> Optional[str]: - """Return the target's OS version. + if issubclass(plugincls, OSPlugin): + # Also store the OS plugins in a tree by module path + # This is used to filter plugins based on the OSPlugin subclass + # We don't store anything at the end of the tree, as we only use the tree to check if a plugin is compatible - Returns: - The OS version as string. - """ - raise NotImplementedError + # Also slightly modify the module key to allow for more efficient filtering later + # This is done by removing the last two parts of the module key, which are the file name and the class name + module_parts = module_key.split(".") + if module_parts[-2] != "_os": + log.warning("OS plugin modules should be named as /_os.py: %s", module_key) - @export(record=EmptyRecord) - def users(self) -> list[Record]: - """Return the users available in the target. + obj = PLUGINS.__ostree__ + for part in module_parts[:-2]: + obj = obj.setdefault(part, {}) - Returns: - A list of user records. - """ - raise NotImplementedError + log.debug("Plugin registered: %s", module_key) - @export(property=True) - def os(self) -> str: - """Return a slug of the target's OS name. - Returns: - A slug of the OS name, e.g. 'windows' or 'linux'. - """ - raise NotImplementedError +def _get_plugins() -> PluginRegistry: + """Load the plugin registry, or generate it if it doesn't exist yet.""" + global PLUGINS, GENERATED - @export(property=True) - def architecture(self) -> Optional[str]: - """Return a slug of the target's OS architecture. + if not GENERATED: + try: + from dissect.target.plugins._pluginlist import PLUGINS + except ImportError: + PLUGINS = generate() - Returns: - A slug of the OS architecture, e.g. 'x86_32-unix', 'MIPS-linux' or - 'AMD64-win32', or 'unknown' if the architecture is unknown. - """ - raise NotImplementedError + GENERATED = True + return PLUGINS -class ChildTargetPlugin(Plugin): - """A Child target is a special plugin that can list more Targets. - For example, :class:`~dissect.target.plugins.child.esxi.ESXiChildTargetPlugin` can - list all of the Virtual Machines on the host. - """ +def _module_path(cls: type[Plugin] | str) -> str: + """Returns the module path relative to ``dissect.target.plugins``.""" + if issubclass(cls, Plugin): + module = getattr(cls, "__module__", "") + elif isinstance(cls, str): + module = cls + else: + raise ValueError(f"Invalid argument type: {cls}") - __type__ = None + return module.replace(MODULE_PATH, "").lstrip(".") - def list_children(self) -> Iterator[ChildTargetRecord]: - """Yield :class:`~dissect.target.helpers.record.ChildTargetRecord` records of all - possible child targets on this target. - """ - raise NotImplementedError +def _os_match(osfilter: type[OSPlugin], module_path: str) -> bool: + """Check if the a plugin is compatible with the given OS filter.""" + if issubclass(osfilter, default._os.DefaultPlugin): + return True -def register(plugincls: Type[Plugin]) -> None: - """Register a plugin, and put related data inside :attr:`PLUGINS`. + os_parts = _module_path(osfilter).split(".")[:-1] - This function uses the following private attributes that are set using decorators: - - ``__exported__``: Set in :func:`export`. - - ``__internal__``: Set in :func:`internal`. + obj = _get_plugins().__ostree__ + for plugin_part, os_part in zip_longest(module_path.split("."), os_parts): + if plugin_part not in obj: + break - Additionally, ``register`` sets the following private attributes on the `plugincls`: - - ``__plugin__``: Always set to ``True``. - - ``__functions__``: A list of all the methods and properties that are ``__internal__`` or ``__exported__``. - - ``__exports__``: A list of all the methods or properties that were explicitly exported. + if plugin_part != os_part: + return False - Args: - plugincls: A plugin class to register. + obj = obj[plugin_part] - Raises: - ValueError: If ``plugincls`` is not a subclass of :class:`Plugin`. - """ - if not issubclass(plugincls, Plugin): - raise ValueError("Not a subclass of Plugin") + return True - exports = [] - functions = [] - # First pass to resolve aliases - for attr in get_nonprivate_attributes(plugincls): - for alias in getattr(attr, "__aliases__", []): - clone_alias(plugincls, attr, alias) +def plugins(osfilter: type[OSPlugin] | None = None, *, index: str = "__regular__") -> Iterator[PluginDescriptor]: + """Walk the plugin registry and return plugin descriptors. - for attr in get_nonprivate_attributes(plugincls): - if isinstance(attr, property): - attr = attr.fget + If ``osfilter`` is specified, only plugins related to the provided OSPlugin, or plugins + with no OS relation are returned. If ``osfilter`` is ``None``, all plugins will be returned. - if getattr(attr, "__autogen__", False) and plugincls != plugincls.__nsplugin__: - continue + One exception to this is if the ``osfilter`` is a (sub-)class of DefaultPlugin, then plugins + are returned as if no ``osfilter`` was specified. - if getattr(attr, "__exported__", False): - exports.append(attr.__name__) - functions.append(attr.__name__) + The ``index`` parameter can be used to specify the index to return plugins from. By default, + this is set to return regular plugins. Other possible values are ``__os__`` and ``__child__``. + These return :class:`OSPlugin` and :class:`ChildTargetPlugin` respectively. - if getattr(attr, "__internal__", False): - functions.append(attr.__name__) - - plugincls.__plugin__ = True - plugincls.__functions__ = functions - plugincls.__exports__ = exports + Args: + osfilter: The optional :class:`OSPlugin` to filter the returned plugins on. + index: The plugin index to return plugins from. Defaults to regular plugins. - modpath = _modulepath(plugincls) - lookup_path = modpath - if modpath.endswith("._os"): - lookup_path, _, _ = modpath.rpartition(".") + Yields: + Plugin descriptors in the plugin registry based on the given filter criteria. + """ - root = _traverse(lookup_path, PLUGINS) + plugin_index: dict[str, PluginDescriptor] = getattr(_get_plugins().__plugins__, index, {}) - log.debug("Plugin registered: %s.%s", plugincls.__module__, plugincls.__qualname__) + # This is implemented as a list comprehension for performance reasons! + yield from ( + descriptor + for module_path, descriptor in plugin_index.items() + if (index != "__os__" and (osfilter is None or _os_match(osfilter, module_path))) + or (index == "__os__" and (osfilter is None or osfilter.__module__ == descriptor.module)) + ) - if issubclass(plugincls, (OSPlugin, ChildTargetPlugin)): - if issubclass(plugincls, OSPlugin): - special_key = "_os" - elif issubclass(plugincls, ChildTargetPlugin): - special_key = "_child" - root[special_key] = {} - root = root[special_key] +def os_plugins() -> Iterator[PluginDescriptor]: + """Retrieve all OS plugin descriptors.""" + yield from plugins(index="__os__") - # Check if the plugin was already registered - if "class" in root and root["class"] == plugincls.__name__: - return - # Finally register the plugin - root["class"] = plugincls.__name__ - root["module"] = modpath - root["functions"] = plugincls.__functions__ - root["exports"] = plugincls.__exports__ - root["namespace"] = plugincls.__namespace__ - root["fullname"] = ".".join((plugincls.__module__, plugincls.__qualname__)) - root["is_osplugin"] = issubclass(plugincls, OSPlugin) +def child_plugins() -> Iterator[PluginDescriptor]: + """Retrieve all child plugin descriptors.""" + yield from plugins(index="__child__") -def internal(*args, **kwargs) -> Callable: - """Decorator to be used on plugin functions that should be internal only. +def functions(osfilter: type[OSPlugin] | None = None, *, index: str = "__regular__") -> Iterator[FunctionDescriptor]: + """Retrieve all function descriptors. - Making a plugin internal means that it's only callable from the Python API and not through ``target-query``. + Args: + osfilter: The optional :class:`OSPlugin` to filter the returned functions on. + index: The plugin index to return functions from. Defaults to regular functions. - This decorator adds the ``__internal__`` private attribute to a method or property. - The attribute is always set to ``True``, to tell :func:`register` that it is an internal - method or property. + Yields: + Function descriptors in the plugin registry based on the given filter criteria. """ - def decorator(obj): - obj.__internal__ = True - if kwargs.get("property", False): - obj = property(obj) - return obj + function_index: dict[str, dict[str, FunctionDescriptor]] = getattr(_get_plugins().__functions__, index, {}) - if len(args) == 1: - return decorator(args[0]) - else: - return decorator + # This is implemented as a list comprehension for performance reasons! + yield from ( + descriptor + for function_index in function_index.values() + for module_path, descriptor in function_index.items() + if osfilter is None or _os_match(osfilter, module_path) + ) -def arg(*args, **kwargs) -> Callable: - """Decorator to be used on Plugin functions that accept additional command line arguments. +def lookup( + function_name: str, osfilter: type[OSPlugin] | None = None, *, index: str = "__regular__" +) -> Iterator[FunctionDescriptor]: + """Lookup a function descriptor by function name. - Command line arguments can be added using the ``@arg`` decorator. - Arguments to this decorator are directly forwarded to the ``ArgumentParser.add_argument`` function of ``argparse``. - Resulting arguments are passed to the function using kwargs. - The keyword argument name must match the argparse argument name. + Args: + func_name: Function name to lookup. + osfilter: The optional ``OSPlugin`` to filter results with for compatibility. + index: The plugin index to return plugins from. Defaults to regular functions. - This decorator adds the ``__args__`` private attribute to a method or property. - This attribute holds all the command line arguments that were added to the plugin function. + Yields: + Function descriptors that match the given function name and filter criteria. """ - def decorator(obj): - if not hasattr(obj, "__args__"): - obj.__args__ = [] - arglist = getattr(obj, "__args__", []) - arglist.append((args, kwargs)) - return obj + function_index: dict[str, FunctionDescriptor] = getattr(_get_plugins().__functions__, index, {}).get( + function_name, {} + ) - return decorator + # This is implemented as a list comprehension for performance reasons! + entries: Iterator[FunctionDescriptor] = ( + value for key, value in function_index.items() if osfilter is None or _os_match(osfilter, key) + ) + yield from sorted(entries, key=lambda x: x.module.count("."), reverse=True) -def alias(*args, **kwargs: dict[str, Any]) -> Callable: - """Decorator to be used on :class:`Plugin` functions to register an alias of that function.""" - if not kwargs.get("name") and not args: - raise ValueError("Missing argument 'name'") +def load(desc: FunctionDescriptor | PluginDescriptor) -> type[Plugin]: + """Helper function that loads a plugin from a given function or plugin descriptor. - def decorator(obj: Callable) -> Callable: - if not hasattr(obj, "__aliases__"): - obj.__aliases__ = [] + Args: + desc: Function descriptor as returned by :func:`plugin.lookup` or plugin descriptor + as returned by :func:`plugin.plugins`. - if name := (kwargs.get("name") or args[0]): - obj.__aliases__.append(name) + Returns: + The plugin class. + + Raises: + PluginError: Raised when any other exception occurs while trying to load the plugin. + """ + module = desc.module + try: + obj = importlib.import_module(module) + for part in desc.qualname.split("."): + obj = getattr(obj, part) return obj + except Exception as e: + raise PluginError(f"An exception occurred while trying to load a plugin: {module}") from e - return decorator +def os_match(target: Target, descriptor: PluginDescriptor) -> bool: + """Check if a plugin descriptor is compatible with the target OS. -def clone_alias(cls: type, attr: Callable, alias: str) -> None: - """Clone the given attribute to an alias in the provided class.""" + Args: + target: The target to check compatibility with. + descriptor: The plugin descriptor to check compatibility for. + """ + return _os_match(target._os_plugin, f"{descriptor.module}.{descriptor.qualname}") - # Clone the function object - clone = type(attr)(attr.__code__, attr.__globals__, alias, attr.__defaults__, attr.__closure__) - clone.__kwdefaults__ = attr.__kwdefaults__ - # Copy some attributes - functools.update_wrapper(clone, attr) - if wrapped := getattr(attr, "__wrapped__", None): - # update_wrapper sets a new wrapper, we want the original - clone.__wrapped__ = wrapped +def failed() -> list[FailureDescriptor]: + """Return all plugins that failed to load.""" + return _get_plugins().__failed__ - # Update module path so we can fool inspect.getmodule with subclassed Plugin classes - clone.__module__ = cls.__module__ - # Update the names - clone.__name__ = alias - clone.__qualname__ = f"{cls.__name__}.{alias}" +@functools.cache +def _generate_long_paths() -> dict[str, FunctionDescriptor]: + """Generate a dictionary of all long paths to their function descriptors.""" + paths = {} + for value in _get_plugins().__functions__.__regular__.values(): + for descriptor in value.values(): + # Namespace plugins are callable so exclude the explicit __call__ method + if descriptor.method_name == "__call__": + continue + paths[descriptor.path] = descriptor - setattr(cls, alias, clone) + return paths -def plugins( - osfilter: Optional[type[OSPlugin]] = None, - special_keys: set[str] = set(), - only_special_keys: bool = False, -) -> Iterator[PluginDescriptor]: - """Walk the ``PLUGINS`` tree and return plugins. +def find_functions( + patterns: str, + target: Target | None = None, + compatibility: bool = False, + ignore_load_errors: bool = False, + show_hidden: bool = False, +) -> tuple[list[FunctionDescriptor], set[str]]: + """Finds exported plugin functions that match the target and the patterns. - If ``osfilter`` is specified, only plugins related to the provided - OSPlugin, or plugins with no OS relation are returned. - If ``osfilter`` is ``None``, all plugins will be returned. + Given a target, a comma separated list of patterns and an optional compatibility flag, + this function finds matching plugins, optionally checking compatibility. - One exception to this is if the ``osfilter`` is a (sub-)class of - DefaultPlugin, then plugins are returned as if no ``osfilter`` was - specified. + Returns: + A tuple containing a list of matching function descriptors and a set of invalid patterns. + """ + found = [] - Another exeption to this are plugins in the ``PLUGINS`` tree which are - under a key that starts with a '_'. Those are only returned if their exact - key is specified in ``special_keys``. + __functions__ = _get_plugins().__functions__ - An exception to these exceptions is in the case of ``OSPlugin`` (sub-)class - plugins and ``os_filter`` is not ``None``. These plugins live in the - ``PLUGINS`` tree under the ``_os`` special key. Those plugins are only - returned if they fully match the provided ``osfilter``. + regular_functions = __functions__.__regular__ + os_functions = __functions__.__os__ - The ``only_special_keys`` option returns only the plugins which are under a - special key that is defined in ``special_keys``. All filtering here will - happen as stated in the above cases. + os_filter = target._os_plugin if target is not None else None - Args: - osfilter: The optional OSPlugin to filter the returned plugins on. - special_keys: Also return plugins which are under the special ('_') keys in this set. - only_special_keys: Only return the plugins under the keys in ``special_keys`` and no others. + invalid_functions = set() - Yields: - Plugins in the ``PLUGINS`` tree based on the given filter criteria. - """ - - if osfilter is not None: - # The PLUGINS tree does not include the hierarchy up to the plugins - # directory (dissect.target.plugins) for the built-in plugins. For the - # plugins in the directory specified in --plugin-path, the hierarchy - # starts at that directory. - # - # Built-in OSPlugins do have the dissect.target.plugins path in their - # module name, so it needs to be stripped, e.g.: - # dissect.target.plugins.general.default -> general.default - # dissect.target.plugins.windows._os -> plugins.windows._os - # - # The module name of OSPlugins from --plugin-path starts at the - # directory specified in that option, e.g.: - # --plugin-path=/some/path/, with a file foo/baros/_os.py - # will have a module name of: foo.baros._os - filter_path = _modulepath(osfilter).split(".") - - # If an OSPlugin is not defined in a file called _os.py, an extra `_os` - # part is added to the PLUGINS tree. - # For example the default OS plugin with module name general.default - # (after stripping of the build-in hierarchy) will be added at: - # general - # \- default - # \- _os - # However the `_os` part is not in the module name. Modules that are - # defined in an _os.py file have the `_os` part in their module name. - # It is stripped out, so the filter is similar for both types of - # OSPlugin files. - if filter_path[-1] == "_os": - filter_path = filter_path[:-1] - else: - filter_path = [] - - def _walk( - root: dict, - special_keys: set[str] = set(), - only_special_keys: bool = False, - prev_module_path: list[str] = [], - ): - for key, obj in root.items(): - module_path = prev_module_path.copy() - module_path.append(key) - - # A branch in the PLUGINS tree is traversed to the next level if: - # - there are no filters (which in effect means all plugins are - # returned including all _os plugins). - # - the osfilter is the default plugin (which means all normal plugins but - # only the default _os plugin is returned). - # - there is no _os plugin on the next level (we're traversing a - # "normal" plugin branch or already jumped into an OS specific - # branch because of a filter_path match) - # - the current module_path fully matches the (beginning of) the - # filter path (this allows traversing into the specific os branch - # for the given os filter and any sub branches which are not os - # branches (of a sub-os) themselves). - if ( - not filter_path - or issubclass(osfilter, general.default.DefaultPlugin) - or "_os" not in obj - or module_path == filter_path[: len(module_path)] - ): - if key.startswith("_"): - if key in special_keys: - # OSPlugins are treated special and are only returned - # if their module_path matches the full filter_path. - # - # Note that the module_path includes the `_os` part, - # which may have been explicitly added in the - # hierarchy. This part needs to be stripped out when - # matching against the filter_path, where it was either - # not present or stripped out. - if key != "_os" or ( - key == "_os" and (not filter_path or (filter_path and module_path[:-1] == filter_path)) - ): - # If the special key is a leaf-node, we just give it back. - # If it is a branch, we give back the full branch, - # not just the special_keys if only_special_keys - # was set to True. - if "functions" in obj: - yield obj - else: - yield from _walk( - obj, - special_keys=special_keys, - only_special_keys=False, - prev_module_path=module_path, - ) - else: - continue - else: - continue + for pattern in patterns.split(","): + if not pattern: + continue - else: - if "functions" in obj: - if not (special_keys and only_special_keys): - yield obj - else: - yield from _walk( - obj, - special_keys=special_keys, - only_special_keys=only_special_keys, - prev_module_path=module_path, - ) - - yield from sorted( - _walk( - _get_plugins(), - special_keys=special_keys, - only_special_keys=only_special_keys, - ), - key=lambda plugin: len(plugin["module"]), - reverse=True, - ) + exact_match = pattern in regular_functions + exact_os_match = pattern in os_functions + if exact_match or exact_os_match: + if matches := list(_filter_exact_match(pattern, os_filter, exact_match, exact_os_match)): + found.extend(matches) + else: + invalid_functions.add(pattern) + else: + # If we don't have an exact function match, do a slower treematch + if matches := list(_filter_tree_match(pattern, os_filter, show_hidden=show_hidden)): + found.extend(matches) + else: + invalid_functions.add(pattern) -def os_plugins() -> Iterator[PluginDescriptor]: - """Retrieve all OS plugin descriptors.""" - yield from plugins(special_keys={"_os"}, only_special_keys=True) + if compatibility and target is not None: + result = list(_filter_compatible(found, target, ignore_load_errors)) + else: + result = found + return result, invalid_functions -def child_plugins() -> Iterator[PluginDescriptor]: - """Retrieve all child plugin descriptors.""" - yield from plugins(special_keys={"_child"}, only_special_keys=True) +def _filter_exact_match( + pattern: str, os_filter: str, exact_match: bool, exact_os_match: bool +) -> Iterator[FunctionDescriptor]: + if exact_match: + descriptors = lookup(pattern, os_filter) + elif exact_os_match: + descriptors = lookup(pattern, os_filter, index="__os__") -def lookup(func_name: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Lookup a plugin descriptor by function name. + for descriptor in descriptors: + if not descriptor.exported: + continue - Args: - func_name: Function name to lookup. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - yield from get_plugins_by_func_name(func_name, osfilter=osfilter) - yield from get_plugins_by_namespace(func_name, osfilter=osfilter) + yield descriptor -def get_plugins_by_func_name(func_name: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Get a plugin descriptor by function name. +def _filter_tree_match(pattern: str, os_filter: str, show_hidden: bool = False) -> Iterator[FunctionDescriptor]: + path_lookup = _generate_long_paths() - Args: - func_name: Function name to lookup. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - for plugin_desc in plugins(osfilter): - if not plugin_desc["namespace"] and func_name in plugin_desc["functions"]: - yield plugin_desc + # Change the treematch pattern into an fnmatch-able pattern to give back all functions from the sub-tree + # (if there is a subtree). + # + # Examples: + # -f apps.webservers.iis -> apps.webservers.iis* (logs etc) + # -f apps.webservers.iis.logs -> apps.webservers.iis.logs* (only the logs, there is no subtree) + # We do not include a dot because that does not work if the full path is given: + # -f apps.webservers.iis.logs != apps.webservers.iis.logs.* (does not work) + search_pattern = pattern + if not has_glob_magic(pattern): + search_pattern += "*" + for path in fnmatch.filter(path_lookup.keys(), search_pattern): + descriptor = path_lookup[path] -def get_plugins_by_namespace(namespace: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Get a plugin descriptor by namespace. + # Skip plugins that don't want to be found by wildcards + if not descriptor.exported or (not show_hidden and not descriptor.findable): + continue - Args: - namespace: Plugin namespace to match. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - for plugin_desc in plugins(osfilter): - if namespace == plugin_desc["namespace"]: - yield plugin_desc + # Skip plugins that do not match our OS + if os_filter and not _os_match(os_filter, descriptor.path): + continue + yield descriptor -def load(plugin_desc: PluginDescriptor) -> Type[Plugin]: - """Helper function that loads a plugin from a given plugin description. - Args: - plugin_desc: Plugin description as returned by plugin.lookup(). +def _filter_compatible( + descriptors: list[FunctionDescriptor], target: Target, ignore_load_errors: bool = False +) -> Iterator[FunctionDescriptor]: + """Filter a list of function descriptors based on compatibility with a target.""" + seen = set() + for descriptor in descriptors: + try: + plugincls = load(descriptor) + except Exception: + if ignore_load_errors: + continue + raise - Returns: - The plugin class. + if plugincls not in seen: + try: + if not plugincls(target).is_compatible(): + continue + except Exception: + continue - Raises: - PluginError: Raised when any other exception occurs while trying to load the plugin. - """ - module = plugin_desc["fullname"].rsplit(".", 1)[0] + yield descriptor - try: - module = importlib.import_module(module) - return getattr(module, plugin_desc["class"]) - except Exception as e: - raise PluginError(f"An exception occurred while trying to load a plugin: {module}") from e +def generate() -> dict[str, Any]: + """Internal function to generate the list of available plugins. -def failed() -> list[dict[str, Any]]: - """Return all plugins that failed to load.""" - return _get_plugins().get("_failed", []) + Walks the plugins directory and imports any ``.py`` files in there. + Plugins will be automatically registered. + Returns: + The global ``PLUGINS`` dictionary. + """ + plugins_dir = Path(__file__).parent / "plugins" + for path in _find_py_files(plugins_dir): + relative_path = path.relative_to(plugins_dir) + module_tuple = (MODULE_PATH, *relative_path.parent.parts, relative_path.stem) + load_module_from_name(".".join(module_tuple)) -def _get_plugins() -> dict[str, PluginDescriptor]: - """Load the plugin registry, or generate it if it doesn't exist yet.""" - global PLUGINS, GENERATED - if not GENERATED: - PLUGINS = generate() - GENERATED = True return PLUGINS -def save_plugin_import_failure(module: str) -> None: - """Store errors that occurred during plugin import.""" - if "_failed" not in PLUGINS: - PLUGINS["_failed"] = [] - - stacktrace = traceback.format_exception(*sys.exc_info()) - PLUGINS["_failed"].append( - { - "module": module, - "stacktrace": stacktrace, - } - ) - - -def find_py_files(plugin_path: Path) -> Iterator[Path]: - """Walk all the files and directories in ``plugin_path`` and return all files ending in ``.py``. +def _find_py_files(path: Path) -> Iterator[Path]: + """Walk all the files and directories in ``path`` and return all files ending in ``.py``. Do not walk or yield paths containing the following names: - __pycache__ - __init__ - Furthermore, it logs an error if ``plugin_path`` does not exist. + Furthermore, it logs an error if ``path`` does not exist. Args: - plugin_path: The path to a directory or file to walk and filter. + path: The path to a directory or file to walk and filter. """ - if not plugin_path.exists(): - log.error("Path %s does not exist.", plugin_path) + if not path.exists(): + log.error("Path %s does not exist.", path) return - if plugin_path.is_file(): - path_iterator = [plugin_path] + if path.is_file(): + it = [path] else: - path_iterator = plugin_path.glob("**/*.py") + it = path.glob("**/*.py") - for path in path_iterator: - if not path.is_file() or str(path).endswith("__init__.py"): + for entry in it: + if not entry.is_file() or entry.name == "__init__.py": continue - yield path + yield entry def load_module_from_name(module_path: str) -> None: @@ -875,33 +959,10 @@ def load_module_from_name(module_path: str) -> None: except Exception as e: log.info("Unable to import %s", module_path) log.debug("Error while trying to import module %s", module_path, exc_info=e) - save_plugin_import_failure(module_path) - - -def generate() -> dict[str, Any]: - """Internal function to generate the list of available plugins. - - Walks the plugins directory and imports any .py files in there. - Plugins will be automatically registered due to the decorators on them. - - Returns: - The global ``PLUGINS`` dictionary. - """ - global PLUGINS - - if "_failed" not in PLUGINS: - PLUGINS["_failed"] = [] - - plugins_dir = Path(__file__).parent / "plugins" - for path in find_py_files(plugins_dir): - relative_path = path.relative_to(plugins_dir) - module_tuple = (MODULE_PATH, *relative_path.parent.parts, relative_path.stem) - load_module_from_name(".".join(module_tuple)) - - return PLUGINS + _save_plugin_import_failure(module_path) -def load_module_from_file(path: Path, base_path: Path): +def load_module_from_file(path: Path, base_path: Path) -> None: """Loads a module from a file indicated by ``path`` relative to ``base_path``. The module is added to ``sys.modules`` so it can be found everywhere. @@ -920,25 +981,26 @@ def load_module_from_file(path: Path, base_path: Path): except Exception as e: log.error("Unable to import %s", path) log.debug("Error while trying to import module %s", path, exc_info=e) - save_plugin_import_failure(str(path)) + _save_plugin_import_failure(str(path)) -def load_modules_from_paths(plugin_dirs: list[Path]) -> None: - """Iterate over the ``plugin_dirs`` and load all ``.py`` files.""" - for plugin_path in plugin_dirs: - for path in find_py_files(plugin_path): - base_path = plugin_path.parent if path == plugin_path else plugin_path - load_module_from_file(path, base_path) +def load_modules_from_paths(paths: list[Path]) -> None: + """Iterate over the ``paths`` and load all ``.py`` files.""" + for path in paths: + for file in _find_py_files(path): + base_path = path.parent if file == path else path + load_module_from_file(file, base_path) def get_external_module_paths(path_list: list[Path]) -> list[Path]: - """Create a deduplicated list of paths.""" + """Return a list of external plugin directories.""" output_list = environment_variable_paths() + path_list return list(set(output_list)) def environment_variable_paths() -> list[Path]: + """Return additional plugin directories specified by the ``DISSECT_PLUGINS`` environment variable.""" env_var = os.environ.get("DISSECT_PLUGINS") plugin_dirs = env_var.split(":") if env_var else [] @@ -946,403 +1008,338 @@ def environment_variable_paths() -> list[Path]: return [Path(directory) for directory in plugin_dirs] -def _traverse(key: str, obj: dict[str, Any]) -> dict[str, Any]: - """Split a module path up in a dictionary.""" - for p in key.split("."): - if p not in obj: - obj[p] = {} +def _save_plugin_import_failure(module: str) -> None: + """Store errors that occurred during plugin import.""" + stacktrace = traceback.format_exception(*sys.exc_info()) + PLUGINS.__failed__.append(FailureDescriptor(module, stacktrace)) - obj = obj[p] - return obj +def _get_nonprivate_attribute_names(cls: type[Plugin]) -> list[str]: + """Retrieve all attributes that do not start with ``_``.""" + return [attr for attr in dir(cls) if not attr.startswith("_")] -def _modulepath(cls) -> str: - """Returns the module path of a :class:`Plugin` relative to ``dissect.target.plugins``.""" - module = getattr(cls, "__module__", "") - return module.replace(MODULE_PATH, "").lstrip(".") +def _get_nonprivate_attributes(cls: type[Plugin]) -> list[Any]: + """Retrieve all public attributes of a :class:`Plugin`.""" + # Note: `dir()` might return attributes from parent class + return [getattr(cls, attr) for attr in _get_nonprivate_attribute_names(cls)] -# These need to be at the bottom of the module because __init_subclass__ requires everything -# in the parent class Plugin to be defined and resolved. -class NamespacePlugin(Plugin): - def __init__(self, target: Target): - """A namespace plugin provides services to access functionality from a group of subplugins. +def _get_nonprivate_methods(cls: type[Plugin]) -> list[Callable]: + """Retrieve all public methods of a :class:`Plugin`.""" + return [attr for attr in _get_nonprivate_attributes(cls) if not isinstance(attr, property) and callable(attr)] - Support is currently limited to shared exported functions that yield records. - """ - super().__init__(target) - # The code below only applies to the direct subclass, indirect subclasses are finished here. - if self.__class__ != self.__nsplugin__: - return +def _get_descriptors_on_nonprivate_methods(cls: type[Plugin]) -> list[RecordDescriptor]: + """Return record descriptors set on nonprivate methods in ``cls`` class.""" + descriptors = set() - self._subplugins = [] - for entry in self.SUBPLUGINS: - try: - subplugin = getattr(self.target, entry) - self._subplugins.append(subplugin) - except UnsupportedPluginError: - target.log.warning("Subplugin %s is not compatible with target.", entry) - except Exception: - target.log.exception("Failed to load subplugin: %s", entry) + for m in _get_nonprivate_methods(cls): + descriptors.update(_get_descriptors_on_method(m)) - def check_compatible(self) -> None: - if not len(self._subplugins): - raise UnsupportedPluginError("No compatible subplugins found") + return list(descriptors) - def __init_subclass_namespace__(cls, **kwargs): - # If this is a direct subclass of a Namespace plugin, create a reference to the current class for indirect - # subclasses. This is necessary to autogenerate aggregate methods there - cls.__nsplugin__ = cls - cls.__findable__ = False - def __init_subclass_subplugin__(cls, **kwargs): - cls.__findable__ = True +def _get_descriptors_on_method(method: Callable) -> list[RecordDescriptor]: + """Return record descriptors set on a method.""" + if not (record := getattr(method, "__record__", None)): + return [] - if not getattr(cls.__nsplugin__, "SUBPLUGINS", None): - cls.__nsplugin__.SUBPLUGINS = set() + try: + # check if __record__ value is iterable (for example, a list) + return list(record) + except TypeError: + return [record] - # Register the current plugin class as a subplugin with - # the direct subclass of NamespacePlugin - cls.__nsplugin__.SUBPLUGINS.add(cls.__namespace__) - # Generate a tuple of class names for which we do not want to add subplugin functions, which is the - # namespaceplugin and all of its superclasses (minus the base object). - reserved_cls_names = tuple({_class.__name__ for _class in cls.__nsplugin__.mro() if _class is not object}) +# Classes for specific types of plugins +# These need to be at the bottom of the module because __init_subclass__ requires everything +# in the parent class Plugin to be defined and resolved. +class OSPlugin(Plugin): + """Base class for OS plugins. - # Collect the public attrs of the subplugin - for subplugin_func_name in cls.__exports__: - subplugin_func = inspect.getattr_static(cls, subplugin_func_name) + This provides a base class for certain common functions of OS's, which each OS plugin has to implement separately. - # The attr need to be callable and exported - if not isinstance(subplugin_func, Callable): - continue + For example, it provides an interface for retrieving the hostname and users of a target. - # The method needs to output records - if getattr(subplugin_func, "__output__", None) not in ["record", "yield"]: - continue + All derived classes MUST implement ALL the classmethods and exported + methods with the same ``@classmethod`` or ``@export(...)`` annotation. + """ - # The method may not be part of a parent class. - if subplugin_func.__qualname__.startswith(reserved_cls_names): - continue + def __init_subclass__(cls, **kwargs): + # Note that cls is the subclass + super().__init_subclass__(**kwargs) - # If we already have an aggregate method, skip - if existing_aggregator := getattr(cls.__nsplugin__, subplugin_func_name, None): - if not hasattr(existing_aggregator, "__subplugins__"): - # This is not an aggregator, but a re-implementation of a subclass function by the subplugin. - continue - existing_aggregator.__subplugins__.append(cls.__namespace__) - continue + for os_method in _get_nonprivate_attributes(OSPlugin): + if isinstance(os_method, property): + os_method = os_method.fget + os_docstring = os_method.__doc__ - # The generic template for the aggregator method - def generate_aggregator(method_name: str) -> Callable: - def aggregator(self) -> Iterator[Record]: - for entry in aggregator.__subplugins__: - try: - subplugin = getattr(self.target, entry) - yield from getattr(subplugin, method_name)() - except UnsupportedPluginError: - continue - except Exception as e: - self.target.log.error("Subplugin: %s raised an exception for: %s", entry, method_name) - self.target.log.debug("Exception: %s", e, exc_info=e) - - # Holds the subplugins that share this method - aggregator.__subplugins__ = [] - - return aggregator - - # The generic template for the documentation method - def generate_documentor(cls, method_name: str, aggregator: Callable) -> str: - def documentor(): - return defaultdict( - lambda: "???", - { - "func_name": f"{cls.__nsplugin__.__namespace__}.{method_name}", - "short_description": "".join( - [ - f"Return {method_name} for: ", - ",".join(aggregator.__subplugins__), - ] - ), - "output_type": "records", - }, - ) + method = getattr(cls, os_method.__name__, None) + if isinstance(method, property): + method = method.fget + # This works as None has a __doc__ property (which is None). + docstring = method.__doc__ - return documentor + if method and not docstring: + if hasattr(method, "__func__"): + method = method.__func__ + method.__doc__ = os_docstring - # Manifacture a method for the namespaced class - generated_aggregator = generate_aggregator(subplugin_func_name) - generated_documentor = generate_documentor(cls, subplugin_func_name, generated_aggregator) + def check_compatible(self) -> bool: + """OSPlugin's use a different compatibility check, override the one from the :class:`Plugin` class. - # Add as an attribute to the namespace class - setattr(cls.__nsplugin__, subplugin_func_name, generated_aggregator) + Returns: + This function always returns ``True``. + """ + return True - # Copy the meta descriptors of the function attribute - for copy_attr in ["__output__", "__record__", "__doc__", "__exported__"]: - setattr(generated_aggregator, copy_attr, getattr(subplugin_func, copy_attr, None)) + @classmethod + def detect(cls, fs: Filesystem) -> Filesystem | None: + """Provide detection of this OSPlugin on a given filesystem. - # Add subplugin to aggregator - generated_aggregator.__subplugins__.append(cls.__namespace__) + Args: + fs: :class:`~dissect.target.filesystem.Filesystem` to detect the OS on. - # Mark the function as being autogenerated - setattr(generated_aggregator, "__autogen__", True) + Returns: + The root filesystem / sysvol when found. + """ + raise NotImplementedError - # Add the documentor function to the aggregator - setattr(generated_aggregator, "get_func_doc_spec", generated_documentor) + @classmethod + def create(cls, target: Target, sysvol: Filesystem) -> OSPlugin: + """Initiate this OSPlugin with the given target and detected filesystem. - # Register the newly auto-created method - cls.__nsplugin__.__exports__.append(subplugin_func_name) - cls.__nsplugin__.__functions__.append(subplugin_func_name) + Args: + target: The :class:`~dissect.target.target.Target` object. + sysvol: The filesystem that was detected in the ``detect()`` function. - def __init_subclass__(cls, **kwargs): - # Upon subclassing, decide whether this is a direct subclass of NamespacePlugin - # If this is not the case, autogenerate aggregate methods for methods record output. - super().__init_subclass__(**kwargs) - if cls.__bases__[0] != NamespacePlugin: - cls.__init_subclass_subplugin__(cls, **kwargs) - else: - cls.__init_subclass_namespace__(cls, **kwargs) + Returns: + An instantiated version of the OSPlugin. + """ + raise NotImplementedError + @export(property=True) + def hostname(self) -> str | None: + """Return the target's hostname. -__COMMON_PLUGIN_METHOD_NAMES__ = {attr.__name__ for attr in get_nonprivate_methods(Plugin)} + Returns: + The hostname as string. + """ + raise NotImplementedError + @export(property=True) + def ips(self) -> list[str]: + """Return the IP addresses configured in the target. -class InternalPlugin(Plugin): - """Parent class for internal plugins. + Returns: + The IPs as list. + """ + raise NotImplementedError - InternalPlugin marks all non-private methods internal by default - (same as ``@internal`` decorator). - """ + @export(property=True) + def version(self) -> str | None: + """Return the target's OS version. - def __init_subclass__(cls, **kwargs): - for method in get_nonprivate_methods(cls): - if callable(method) and method.__name__ not in __COMMON_PLUGIN_METHOD_NAMES__: - method.__internal__ = True + Returns: + The OS version as string. + """ + raise NotImplementedError - super().__init_subclass__(**kwargs) - return cls + @export(record=EmptyRecord) + def users(self) -> list[Record]: + """Return the users available in the target. + + Returns: + A list of user records. + """ + raise NotImplementedError + @export(property=True) + def os(self) -> str: + """Return a slug of the target's OS name. -class InternalNamespacePlugin(NamespacePlugin, InternalPlugin): - pass + Returns: + A slug of the OS name, e.g. 'windows' or 'linux'. + """ + raise NotImplementedError + @export(property=True) + def architecture(self) -> str | None: + """Return a slug of the target's OS architecture. -@dataclass(frozen=True, eq=True) -class PluginFunction: - name: str - path: str - output_type: str - class_object: type[Plugin] - method_name: str - plugin_desc: PluginDescriptor = field(hash=False) + Returns: + A slug of the OS architecture, e.g. 'x86_32-unix', 'MIPS-linux' or + 'AMD64-win32', or 'unknown' if the architecture is unknown. + """ + raise NotImplementedError + + +class ChildTargetPlugin(Plugin): + """A Child target is a special plugin that can list more Targets. + + For example, :class:`~dissect.target.plugins.child.esxi.ESXiChildTargetPlugin` can + list all of the Virtual Machines on the host. + """ + + __type__ = None - def __repr__(self) -> str: - return self.path + def list_children(self) -> Iterator[ChildTargetRecord]: + """Yield :class:`~dissect.target.helpers.record.ChildTargetRecord` records of all + possible child targets on this target. + """ + raise NotImplementedError -def plugin_function_index(target: Optional[Target]) -> tuple[dict[str, PluginDescriptor], set[str]]: - """Returns an index-list for plugins. +class NamespacePlugin(Plugin): + """A namespace plugin provides services to access functionality from a group of subplugins. - This list is used to match CLI expressions against to find the desired plugin. - Also returns the roots to determine whether a CLI expression has to be compared - to the plugin tree or parsed using legacy rules. + Support is currently limited to shared exported functions with output type ``record`` and ``yield``. """ - if target is None: - os_type = None - elif target._os_plugin is None: - os_type = general.default.DefaultPlugin - elif isinstance(target._os_plugin, type) and issubclass(target._os_plugin, OSPlugin): - os_type = target._os_plugin - elif isinstance(target._os_plugin, OSPlugin): - os_type = type(target._os_plugin) - else: - raise TypeError( - "target must be None or target._os_plugin must be either None, " - "a subclass of OSPlugin or an instance of OSPlugin" - ) + def __init_subclass__(cls, **kwargs): + # Upon subclassing, decide whether this is a direct subclass of NamespacePlugin + # If this is not the case, autogenerate aggregate methods for methods record output. - index = {} - rootset = set() + # Do not process the "base" subclasses defined in this file + if cls.__module__ == NamespacePlugin.__module__: + super().__init_subclass__(**kwargs) + return - all_plugins = plugins(osfilter=os_type, special_keys={"_child", "_os"}) + if cls.__bases__[0] not in (NamespacePlugin, InternalNamespacePlugin): + # This is a subclass of a subclass of (Internal)NamespacePlugin + # We need to aggregate the methods of this subclass + cls.__findable__ = True + # Run the normal subclass initialization first so the plugin gets all its exports registered + super().__init_subclass__(**kwargs) + cls.__init_subclass_subplugin__(cls, **kwargs) + cls.__nsplugin__.__init_subclass__(**kwargs) + else: + # This is a direct subclass of (Internal)NamespacePlugin + # Not much to do here, just do some initialization and register the plugin + cls.__findable__ = False + super().__init_subclass__(**kwargs) - for available_original in all_plugins: - # Prevent modifying the global PLUGINS dict, otherwise -f os.windows._os.users fails for instance. - available = available_original.copy() + cls.__nsplugin__ = cls + if not hasattr(cls, "__subplugins__"): + cls.__subplugins__ = set() - modulepath = available["module"] - rootset.add(modulepath.split(".")[0]) + def __init_subclass_subplugin__(cls, **kwargs) -> None: + # Register the current plugin class as a subplugin with + # the direct subclass of NamespacePlugin + cls.__nsplugin__.__subplugins__.add(cls.__namespace__) - if "get_all_records" in available["exports"]: - # The get_all_records does not only need to be not present in the - # index, it also needs to be removed from the exports list, else - # the 'plugins' plugin will still display them. - exports = available["exports"].copy() - exports.remove("get_all_records") - available["exports"] = exports + # Generate a tuple of class names for which we do not want to add subplugin functions, which is the + # NamespacePlugin and all of its superclasses (minus the base object) + reserved_cls = tuple({klass.__name__ for klass in cls.__nsplugin__.mro() if klass is not object}) - if "get_all_records" not in available_original["exports"]: - raise Exception(f"get_all_records removed from {available_original}") + # Collect the public exports of the subplugin + for export_name in cls.__exports__: + export_attr = getattr(cls, export_name) - for exported in available["exports"]: - if available["is_osplugin"] and os_type == general.default.DefaultPlugin: - # This makes the os plugin exports listed under the special - # "OS plugins" header by the 'plugins' plugin. - available["module"] = "" + # The export need to be callable and may not be part of a parent class + if not isinstance(export_attr, Callable) or export_attr.__qualname__.startswith(reserved_cls): + continue - index[f"{modulepath}.{exported}"] = available + # The export needs to output records or yield something else + if (output_type := export_attr.__output__) not in ["record", "yield"]: + continue - return index, rootset + if aggregator := getattr(cls.__nsplugin__, export_name, None): + # If we already have an aggregate method, update some fields + if not hasattr(aggregator, "__subplugins__"): + # This is not an aggregator, but a re-implementation of a subclass method by the subplugin + continue + if aggregator.__output__ != output_type: + # The output type of the existing aggregator is different, so we can't merge them + raise PluginError( + "Cannot merge namespace methods with different output types" + f" ({aggregator.__output__} != {output_type} in {cls.__namespace__}.{export_name})" + ) -def find_plugin_functions( - target: Optional[Target], - patterns: str, - compatibility: bool = False, - **kwargs, -) -> tuple[list[PluginFunction], set[str]]: - """Finds plugins that match the target and the patterns. + # Update record descriptors + if (export_attr.__record__, aggregator.__record__) != (None, None): + aggregator.__record__ = list( + set(_get_descriptors_on_method(export_attr) + _get_descriptors_on_method(aggregator)) + ) + else: + # Generate a method for the parent namespace class + aggregator = export( + output=output_type, + record=export_attr.__record__, + cache=False, + cls=cls.__nsplugin__, + )(cls.__generate_aggregator(cls.__nsplugin__, export_name)) - Given a target, a comma separated list of patterns and an optional compatibility flag, - this function finds matching plugins, optionally checking compatibility and returns - a list of plugin function descriptors (including output types). - """ - result = [] + # Add to the parent namespace class + setattr(cls.__nsplugin__, export_name, aggregator) - functions, rootset = plugin_function_index(target) + # Add subplugin to aggregator + aggregator.__subplugins__.append(cls.__namespace__) + cls.__update_aggregator_docs(aggregator) - invalid_funcs = set() - show_hidden = kwargs.get("show_hidden", False) - ignore_load_errors = kwargs.get("ignore_load_errors", False) + def check_compatible(self) -> None: + at_least_one = False + for entry in self.__subplugins__: + try: + self.target.get_function(entry) + at_least_one = True + except UnsupportedPluginError: + self.target.log.warning("Subplugin %s is not compatible with target", entry) + except Exception as e: + self.target.log.error("Failed to load subplugin: %s", entry) + self.target.log.debug("", exc_info=e) - for pattern in patterns.split(","): - # Backward compatibility fix for namespace-level plugins (i.e. chrome) - # If an exact namespace match is found, the pattern is changed to the tree to that namespace. - # Examples: - # -f browser -> apps.browser.browser - # -f iexplore -> apps.browser.iexplore - namespace_match = False - for index_name, func in functions.items(): - if func["namespace"] == pattern: - pattern = func["module"] - namespace_match = True - break - - wildcard = any(char in pattern for char in ["*", "!", "?", "[", "]"]) - treematch = pattern.split(".")[0] in rootset and pattern != "os" - exact_match = pattern in functions - - # Allow for exact and namespace matches even if the plugin does not want to be found, otherwise you cannot - # reach documented namespace plugins like apps.browser.browser.downloads. - # You can *always* run these using the namespace/classic-style like: browser.downloads (but -l lists them - # in the tree for documentation purposes so it would be misleading not to allow tree access as well). - # - # Note that these tree items will never respond to wildcards though to avoid duplicate results, e.g. when - # querying apps.browser.*, this also means apps.browser.browser.* won't work. - if exact_match or namespace_match: - show_hidden = True - - # Change the treematch pattern into an fnmatch-able pattern to give back all functions from the sub-tree - # (if there is a subtree). - # - # Examples: - # -f browser -> apps.browser.browser* (the whole package, due to a namespace match) - # -f apps.webservers.iis -> apps.webservers.iis* (logs etc) - # -f apps.webservers.iis.logs -> apps.webservers.iis.logs* (only the logs, there is no subtree) - # We do not include a dot because that does not work if the full path is given: - # -f apps.webservers.iis.logs != apps.webservers.iis.logs.* (does not work) - # - # In practice a namespace_match would almost always also be a treematch, except when the namespace plugin - # is in the root of the plugin tree. - if (treematch or namespace_match) and not wildcard and not exact_match: - pattern += "*" - - if wildcard or treematch: - matches = False - for index_name in fnmatch.filter(functions.keys(), pattern): - func = functions[index_name] - - method_name = index_name.split(".")[-1] - try: - loaded_plugin_object = load(func) - except Exception: - if ignore_load_errors: - continue - raise + if not at_least_one: + raise UnsupportedPluginError("No compatible subplugins found") - # Skip plugins that don't want to be found by wildcards - if not show_hidden and not loaded_plugin_object.__findable__: + @staticmethod + def __generate_aggregator(nsklass: type[NamespacePlugin], method_name: str) -> Callable: + def aggregator(self) -> Iterator[Record]: + for ns in aggregator.__subplugins__: + try: + yield from self.target.get_function(f"{ns}.{method_name}")[1]() + except UnsupportedPluginError: continue + except Exception as e: + self.target.log.error("Subplugin %s.%s raised an exception", ns, method_name) + self.target.log.debug("", exc_info=e) - fobject = inspect.getattr_static(loaded_plugin_object, method_name) + # Holds the subplugins that share this method + aggregator.__subplugins__ = [] + aggregator.__generated__ = True - if compatibility: - if target is None: - continue - try: - if not loaded_plugin_object(target).is_compatible(): - continue - except Exception: - continue + # Clone some names and attributes + aggregator.__module__ = nsklass.__module__ + aggregator.__name__ = method_name + aggregator.__qualname__ = f"{nsklass.__name__}.{method_name}" - matches = True - result.append( - PluginFunction( - name=f"{func['namespace']}.{method_name}" if func["namespace"] else method_name, - path=index_name, - class_object=loaded_plugin_object, - method_name=method_name, - output_type=getattr(fobject, "__output__", "none"), - plugin_desc=func, - ) - ) + return aggregator - if not matches: - invalid_funcs.add(pattern) + @staticmethod + def __update_aggregator_docs(aggregator: Callable) -> None: + aggregator.__doc__ = f"Return {aggregator.__name__} for: {', '.join(aggregator.__subplugins__)}" - else: - # otherwise match using ~ classic style - if pattern.find(".") > -1: - namespace, funcname = pattern.split(".", 1) - else: - funcname = pattern - namespace = None - plugin_descriptions = [] - for func_path, func in functions.items(): - nsmatch = namespace and func["namespace"] == namespace and func_path.split(".")[-1] == funcname - fmatch = not namespace and not func["namespace"] and func_path.split(".")[-1] == funcname - if nsmatch or fmatch: - plugin_descriptions.append(func) +__INTERNAL_PLUGIN_METHOD_NAMES__ = {attr.__name__ for attr in _get_nonprivate_methods(Plugin)} - if not plugin_descriptions: - invalid_funcs.add(pattern) - for description in plugin_descriptions: - try: - loaded_plugin_object = load(description) - except Exception: - if ignore_load_errors: - continue - raise +class InternalPlugin(Plugin): + """Parent class for internal plugins. + + InternalPlugin marks all non-private methods internal by default + (same as ``@internal`` decorator). + """ - fobject = inspect.getattr_static(loaded_plugin_object, funcname) + def __init_subclass__(cls, **kwargs): + for method in _get_nonprivate_methods(cls): + if callable(method) and method.__name__ not in __INTERNAL_PLUGIN_METHOD_NAMES__: + method.__internal__ = True - if compatibility and not loaded_plugin_object(target).is_compatible(): - continue + super().__init_subclass__(**kwargs) + return cls - result.append( - PluginFunction( - name=f"{description['namespace']}.{funcname}" if description["namespace"] else funcname, - path=f"{description['module']}.{funcname}", - class_object=loaded_plugin_object, - method_name=funcname, - output_type=getattr(fobject, "__output__", "none"), - plugin_desc=description, - ) - ) - return result, invalid_funcs +class InternalNamespacePlugin(NamespacePlugin, InternalPlugin): + pass diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 94cb80b2f..88b929c7f 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -196,7 +196,6 @@ def check_compatible(self) -> None: if not len(self.access_log_paths) and not len(self.error_log_paths): raise UnsupportedPluginError("No Apache directories found") - @plugin.internal def get_log_paths(self) -> tuple[list[Path], list[Path]]: """ Discover any present Apache log paths on the target system. diff --git a/dissect/target/plugins/apps/webserver/caddy.py b/dissect/target/plugins/apps/webserver/caddy.py index 6347cb738..223ee875a 100644 --- a/dissect/target/plugins/apps/webserver/caddy.py +++ b/dissect/target/plugins/apps/webserver/caddy.py @@ -34,7 +34,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No Caddy paths found") - @plugin.internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/apps/webserver/iis.py b/dissect/target/plugins/apps/webserver/iis.py index 5c37ffcf1..f6c49db46 100644 --- a/dissect/target/plugins/apps/webserver/iis.py +++ b/dissect/target/plugins/apps/webserver/iis.py @@ -76,7 +76,6 @@ def check_compatible(self) -> None: if not self.log_dirs: raise UnsupportedPluginError("No IIS log files found") - @plugin.internal def get_log_dirs(self) -> list[tuple[str, Path]]: log_paths = set() @@ -113,13 +112,11 @@ def get_log_dirs(self) -> list[tuple[str, Path]]: return list(log_paths) - @plugin.internal def iter_log_format_path_pairs(self) -> list[tuple[str, str]]: for log_format, log_dir_path in self.log_dirs: for log_file in log_dir_path.glob("*/*.log"): yield (log_format, log_file) - @plugin.internal def parse_autodetect_format_log(self, path: Path) -> Iterator[BasicRecordDescriptor]: first_line = path.open().readline().decode("utf-8", errors="backslashreplace").strip() if first_line.startswith("#"): @@ -127,7 +124,6 @@ def parse_autodetect_format_log(self, path: Path) -> Iterator[BasicRecordDescrip else: yield from self.parse_iis_format_log(path) - @plugin.internal def parse_iis_format_log(self, path: Path) -> Iterator[BasicRecordDescriptor]: """Parse log file in IIS format and stream log records. @@ -189,7 +185,6 @@ def parse_datetime(date_str: str, time_str: str) -> datetime: def _create_extended_descriptor(self, extra_fields: tuple[tuple[str, str]]) -> TargetRecordDescriptor: return TargetRecordDescriptor(LOG_RECORD_NAME, BASIC_RECORD_FIELDS + list(extra_fields)) - @plugin.internal def parse_w3c_format_log(self, path: Path) -> Iterator[TargetRecordDescriptor]: """Parse log file in W3C format and stream log records. diff --git a/dissect/target/plugins/apps/webserver/nginx.py b/dissect/target/plugins/apps/webserver/nginx.py index f47e6b2d3..c38484be6 100644 --- a/dissect/target/plugins/apps/webserver/nginx.py +++ b/dissect/target/plugins/apps/webserver/nginx.py @@ -30,7 +30,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No NGINX directories found") - @plugin.internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/apps/webserver/webserver.py b/dissect/target/plugins/apps/webserver/webserver.py index 0092c4ca0..7c274ee61 100644 --- a/dissect/target/plugins/apps/webserver/webserver.py +++ b/dissect/target/plugins/apps/webserver/webserver.py @@ -41,7 +41,6 @@ class WebserverPlugin(NamespacePlugin): __namespace__ = "webserver" - __findable__ = False @export(record=[WebserverAccessLogRecord, WebserverErrorLogRecord]) def logs(self) -> Iterator[Union[WebserverAccessLogRecord, WebserverErrorLogRecord]]: diff --git a/dissect/target/plugins/general/osinfo.py b/dissect/target/plugins/general/osinfo.py index f1fb54409..3028fa587 100644 --- a/dissect/target/plugins/general/osinfo.py +++ b/dissect/target/plugins/general/osinfo.py @@ -24,8 +24,6 @@ def check_compatible(self) -> None: def osinfo(self) -> Iterator[Union[OSInfoRecord, GroupedRecord]]: """Yield grouped records with target OS info.""" for os_func in self.target._os.__functions__: - if os_func in ["is_compatible", "get_all_records"]: - continue value = getattr(self.target._os, os_func) record = OSInfoRecord(name=os_func, value=None, _target=self.target) if isinstance(value, Callable) and isinstance(subrecords := value(), Generator): diff --git a/dissect/target/plugins/general/plugins.py b/dissect/target/plugins/general/plugins.py index 08b988266..e114b83ed 100644 --- a/dissect/target/plugins/general/plugins.py +++ b/dissect/target/plugins/general/plugins.py @@ -2,103 +2,146 @@ import json import textwrap -from typing import Iterator, Type from dissect.target import plugin from dissect.target.helpers.docs import INDENT_STEP, get_plugin_overview from dissect.target.plugin import Plugin, arg, export +from dissect.target.plugins.os.default._os import DefaultPlugin -def categorize_plugins(plugins_selection: list[dict] = None) -> dict: - """Categorize plugins based on the module it's from.""" +def generate_functions_overview( + functions: list[plugin.FunctionDescriptor] | None = None, include_docs: bool = False +) -> list[str]: + """Generate a tree list of functions with optional documentation.""" - output_dict = dict() + categorized_plugins = _categorize_functions(functions) + plugin_descriptions = _generate_plugin_tree_overview(categorized_plugins, include_docs) - plugins_selection = plugins_selection or get_exported_plugins() + plugins_list = textwrap.indent( + "\n".join(plugin_descriptions) if plugin_descriptions else "None", + prefix=INDENT_STEP, + ) - for plugin_dict in plugins_selection: - tmp_dict = dictify_module_recursive( - list_of_items=plugin_dict["module"].split("."), - last_value=plugin.load(plugin_dict), + failed_descriptions = [] + failed_items = plugin.failed() + for failed_item in failed_items: + module = failed_item.module + exception = failed_item.stacktrace[-1].rstrip() + failed_descriptions.append( + textwrap.dedent( + f"""\ + Module: {module} + Reason: {exception} + """ + ) ) - update_dict_recursive(output_dict, tmp_dict) - return output_dict + failed_list = textwrap.indent( + "\n".join(failed_descriptions) if failed_descriptions else "None\n", + prefix=INDENT_STEP, + ) + lines = [ + "Available plugins:", + plugins_list, + "", + "Failed to load:", + failed_list, + ] + return "\n".join(lines) + + +def generate_functions_json(functions: list[plugin.FunctionDescriptor] | None = None) -> str: + """Generate a JSON representation of all available functions.""" + + loaded = [] + failed = [] + + for desc in functions or _get_default_functions(): + plugincls = plugin.load(desc) + func = getattr(plugincls, desc.method_name) + docstring = func.__doc__.split("\n\n", 1)[0].strip() if func.__doc__ else None + arguments = [ + { + "name": name[0], + "type": getattr(arg.get("type"), "__name__", None), + "help": arg.get("help"), + "default": arg.get("default"), + } + for name, arg in getattr(func, "__args__", []) + ] -def get_exported_plugins() -> list: - """Returns list of exported plugins.""" - return [p for p in plugin.plugins() if len(p["exports"])] + loaded.append( + { + "name": desc.name, + "output": desc.output, + "description": docstring, + "arguments": arguments, + "path": desc.path, + } + ) + if failures := plugin.failed(): + failed = [{"module": f.module, "stacktrace": "".join(f.stacktrace)} for f in failures] -def dictify_module_recursive(list_of_items: list, last_value: Plugin) -> dict: - """Create a dict from a list of strings. + return json.dumps({"loaded": loaded, "failed": failed}) - The last element inside the list, will point to `last_value` - """ - if len(list_of_items) == 1: - return {list_of_items[0]: last_value} - else: - return {list_of_items[0]: dictify_module_recursive(list_of_items[1:], last_value)} +def _get_default_functions() -> list[plugin.FunctionDescriptor]: + return [f for f in plugin.functions() if f.exported] + [ + f for f in plugin.functions(index="__os__") if f.exported and f.module == DefaultPlugin.__module__ + ] -def update_dict_recursive(source_dict: dict, updated_dict: dict) -> dict: - """Update source dictionary with data in updated_dict.""" - for key, value in updated_dict.items(): - if isinstance(value, dict): - source_dict[key] = update_dict_recursive(source_dict.get(key, {}), value) - else: - source_dict[key] = value - return dict(sorted(source_dict.items())) +def _categorize_functions(functions: list[plugin.FunctionDescriptor] | None = None) -> dict: + """Categorize functions based on its module path.""" + functions = functions or _get_default_functions() + result = {} -def output_plugin_description_recursive( - structure_dict: dict | Plugin, - print_docs: bool, - indentation_step: int = 0, -) -> list[str]: - """Create plugin overview with identations.""" + for desc in functions: + obj = result + parts = desc.path.split(".") - if isinstance(structure_dict, type) and issubclass(structure_dict, Plugin): - return [get_plugin_description(structure_dict, print_docs, indentation_step)] + if not desc.namespace or (desc.namespace and desc.method_name != "__call__"): + parts = parts[:-1] - return get_description_dict(structure_dict, print_docs, indentation_step) + for part in parts[:-1]: + obj = obj.setdefault(part, {}) + if parts[-1] not in obj: + obj[parts[-1]] = plugin.load(desc) -def get_plugin_description( - plugin_class: Type[Plugin], - print_docs: bool, - indentation_step: int, -) -> str: - """Returns plugin_overview with specific indentation.""" - - plugin_overview = get_plugin_overview( - plugin_class=plugin_class, - with_func_docstrings=print_docs, - with_plugin_desc=print_docs, - ) - return textwrap.indent(plugin_overview, prefix=" " * indentation_step) + return dict(sorted(result.items())) -def get_description_dict( - structure_dict: dict, +def _generate_plugin_tree_overview( + plugin_tree: dict | type[Plugin], print_docs: bool, - indentation_step: int, + indent: int = 0, ) -> list[str]: - """Returns a list of indented descriptions.""" - - output_descriptions = [] - for key in structure_dict.keys(): - output_descriptions += [ - textwrap.indent(key + ":", prefix=" " * indentation_step) if key != "" else "OS plugins" - ] + output_plugin_description_recursive( - structure_dict[key], - print_docs, - indentation_step=indentation_step + 2, + """Create plugin overview with identations.""" + + if isinstance(plugin_tree, type) and issubclass(plugin_tree, Plugin): + return [ + textwrap.indent( + get_plugin_overview(plugin_tree, print_docs, print_docs), + prefix=" " * indent, + ) + ] + + result = [] + for key in plugin_tree.keys(): + result.append(textwrap.indent(key + ":", prefix=" " * indent) if key != "" else "OS plugins") + result.extend( + _generate_plugin_tree_overview( + plugin_tree[key], + print_docs, + indent=indent + 2, + ) ) - return output_descriptions + return result class PluginListPlugin(Plugin): @@ -114,71 +157,9 @@ def check_compatible(self) -> None: # https://github.com/fox-it/dissect.target/pull/841 # https://github.com/fox-it/dissect.target/issues/889 @arg("--as-json", dest="as_json", action="store_true") - def plugins(self, plugins: list[Plugin] = None, print_docs: bool = False, as_json: bool = False) -> None: + def plugins(self, print_docs: bool = False, as_json: bool = False) -> None: """Print all available plugins.""" - - dict_plugins = list({p.path: p.plugin_desc for p in plugins}.values()) - categorized_plugins = dict(sorted(categorize_plugins(dict_plugins).items())) - - plugin_descriptions = output_plugin_description_recursive(categorized_plugins, print_docs) - - plugins_list = textwrap.indent( - "\n".join(plugin_descriptions) if plugin_descriptions else "None", - prefix=INDENT_STEP, - ) - - failed_descriptions = [] - failed_items = plugin.failed() - for failed_item in failed_items: - module = failed_item["module"] - exception = failed_item["stacktrace"][-1].rstrip() - failed_descriptions.append( - textwrap.dedent( - f"""\ - Module: {module} - Reason: {exception} - """ - ) - ) - - failed_list = textwrap.indent( - "\n".join(failed_descriptions) if failed_descriptions else "None", - prefix=INDENT_STEP, - ) - - output_lines = [ - "Available plugins:", - plugins_list, - "", - "Failed to load:", - failed_list, - ] - if as_json: - out = {"loaded": list(generate_plugins_json(plugins))} - - if failed_plugins := plugin.failed(): - out["failed"] = [ - {"module": p["module"], "stacktrace": "".join(p["stacktrace"])} for p in failed_plugins - ] - - print(json.dumps(out), end="") - + print(generate_functions_json(), end="") else: - print("\n".join(output_lines)) - - -def generate_plugins_json(plugins: list[Plugin]) -> Iterator[dict]: - """Generates JSON output of a list of :class:`Plugin`.""" - - for p in plugins: - func = getattr(p.class_object, p.method_name) - description = getattr(func, "__doc__", None) - summary = description.split("\n\n", 1)[0].strip() if description else None - - yield { - "name": p.name, - "output": p.output_type, - "description": summary, - "path": p.path, - } + print(generate_functions_overview(include_docs=print_docs)) diff --git a/dissect/target/plugins/os/default/__init__.py b/dissect/target/plugins/os/default/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dissect/target/plugins/general/default.py b/dissect/target/plugins/os/default/_os.py similarity index 93% rename from dissect/target/plugins/general/default.py rename to dissect/target/plugins/os/default/_os.py index 140e9386a..bda3e1bc3 100644 --- a/dissect/target/plugins/general/default.py +++ b/dissect/target/plugins/os/default/_os.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Iterator, Optional if TYPE_CHECKING: from flow.record import Record @@ -44,7 +44,7 @@ def version(self) -> Optional[str]: pass @export(record=EmptyRecord) - def users(self) -> list[Record]: + def users(self) -> Iterator[Record]: yield from () @export(property=True) diff --git a/dissect/target/plugins/general/network.py b/dissect/target/plugins/os/default/network.py similarity index 100% rename from dissect/target/plugins/general/network.py rename to dissect/target/plugins/os/default/network.py diff --git a/dissect/target/plugins/os/unix/bsd/osx/network.py b/dissect/target/plugins/os/unix/bsd/osx/network.py index 5017db750..1b80143c7 100644 --- a/dissect/target/plugins/os/unix/bsd/osx/network.py +++ b/dissect/target/plugins/os/unix/bsd/osx/network.py @@ -5,7 +5,7 @@ from typing import Iterator from dissect.target.helpers.record import MacInterfaceRecord -from dissect.target.plugins.general.network import NetworkPlugin +from dissect.target.plugins.os.default.network import NetworkPlugin from dissect.target.target import Target diff --git a/dissect/target/plugins/os/unix/linux/debian/apt.py b/dissect/target/plugins/os/unix/linux/debian/apt.py index b3e7caf84..7ea961fd6 100644 --- a/dissect/target/plugins/os/unix/linux/debian/apt.py +++ b/dissect/target/plugins/os/unix/linux/debian/apt.py @@ -4,9 +4,10 @@ from typing import Iterator from zoneinfo import ZoneInfo -from dissect.target import Target, plugin +from dissect.target import Target from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import open_decompress +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, @@ -30,7 +31,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No APT files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for Apt. diff --git a/dissect/target/plugins/os/unix/linux/network.py b/dissect/target/plugins/os/unix/linux/network.py index a4e55d296..e23f34de4 100644 --- a/dissect/target/plugins/os/unix/linux/network.py +++ b/dissect/target/plugins/os/unix/linux/network.py @@ -9,7 +9,7 @@ from dissect.target.helpers import configutil from dissect.target.helpers.record import UnixInterfaceRecord from dissect.target.helpers.utils import to_list -from dissect.target.plugins.general.network import NetworkPlugin +from dissect.target.plugins.os.default.network import NetworkPlugin if TYPE_CHECKING: from ipaddress import IPv4Address, IPv4Interface, IPv6Address, IPv6Interface diff --git a/dissect/target/plugins/os/unix/linux/redhat/yum.py b/dissect/target/plugins/os/unix/linux/redhat/yum.py index 6ff7bf2a9..7dbf33cfe 100644 --- a/dissect/target/plugins/os/unix/linux/redhat/yum.py +++ b/dissect/target/plugins/os/unix/linux/redhat/yum.py @@ -1,9 +1,9 @@ import re from typing import Iterator -from dissect.target import plugin from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.utils import year_rollover_helper +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, @@ -27,7 +27,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No Yum files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for CentOS' Yellowdog Updater (Yum). diff --git a/dissect/target/plugins/os/unix/linux/suse/zypper.py b/dissect/target/plugins/os/unix/linux/suse/zypper.py index 090faf330..3ef0f4466 100644 --- a/dissect/target/plugins/os/unix/linux/suse/zypper.py +++ b/dissect/target/plugins/os/unix/linux/suse/zypper.py @@ -1,9 +1,9 @@ from datetime import datetime from typing import Iterator -from dissect.target import plugin from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import open_decompress +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, @@ -24,7 +24,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No zypper files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for SuSE's Zypper. diff --git a/dissect/target/plugins/os/unix/log/audit.py b/dissect/target/plugins/os/unix/log/audit.py index 018a0a435..a8873bf94 100644 --- a/dissect/target/plugins/os/unix/log/audit.py +++ b/dissect/target/plugins/os/unix/log/audit.py @@ -7,7 +7,7 @@ from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import basename, open_decompress from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export, internal +from dissect.target.plugin import Plugin, export AuditRecord = TargetRecordDescriptor( "linux/log/audit", @@ -34,7 +34,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No audit path found") - @internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/os/windows/dpapi/keyprovider/keyprovider.py b/dissect/target/plugins/os/windows/dpapi/keyprovider/keyprovider.py index 00a254fe8..1f73a9205 100644 --- a/dissect/target/plugins/os/windows/dpapi/keyprovider/keyprovider.py +++ b/dissect/target/plugins/os/windows/dpapi/keyprovider/keyprovider.py @@ -2,6 +2,8 @@ class KeyProviderPlugin(InternalNamespacePlugin): + """Key provider plugin for DPAPI.""" + __namespace__ = "_dpapi_keyprovider" def check_compatible(self) -> None: diff --git a/dissect/target/plugins/os/windows/lnk.py b/dissect/target/plugins/os/windows/lnk.py index ceb0cb584..c22b01598 100644 --- a/dissect/target/plugins/os/windows/lnk.py +++ b/dissect/target/plugins/os/windows/lnk.py @@ -166,7 +166,7 @@ def lnk(self, path: str | None = None) -> Iterator[LnkRecord]: lnk_file = Lnk(entry.open()) yield parse_lnk_file(self.target, lnk_file, entry) except Exception as e: - self.target.log.warning("Failed to parse link file %s", lnk_file) + self.target.log.warning("Failed to parse link file %s", entry) self.target.log.debug("", exc_info=e) def lnk_entries(self, path: str | None = None) -> Iterator[TargetPath]: diff --git a/dissect/target/plugins/os/windows/log/evt.py b/dissect/target/plugins/os/windows/log/evt.py index 88cb03804..25da5ee25 100644 --- a/dissect/target/plugins/os/windows/log/evt.py +++ b/dissect/target/plugins/os/windows/log/evt.py @@ -49,8 +49,7 @@ class WindowsEventlogsMixin: EVENTLOG_REGISTRY_KEY = "HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Eventlog" LOGS_DIR_PATH = None - @plugin.internal - def get_logs(self, filename_glob="*") -> list[Path]: + def get_logs(self, filename_glob: str = "*") -> list[Path]: file_paths = [] file_paths.extend(self.get_logs_from_dir(self.LOGS_DIR_PATH, filename_glob=filename_glob)) @@ -67,7 +66,6 @@ def get_logs(self, filename_glob="*") -> list[Path]: return file_paths - @plugin.internal def get_logs_from_dir(self, logs_dir: str, filename_glob: str = "*") -> list[Path]: file_paths = [] logs_dir = self.target.fs.path(logs_dir) @@ -77,7 +75,6 @@ def get_logs_from_dir(self, logs_dir: str, filename_glob: str = "*") -> list[Pat self.target.log.debug("Log files found in '%s': %d", self.LOGS_DIR_PATH, len(file_paths)) return file_paths - @plugin.internal def get_logs_from_registry(self, filename_glob: str = "*") -> list[Path]: # compile glob into case-insensitive regex filename_regex = re.compile(fnmatch.translate(filename_glob), re.IGNORECASE) diff --git a/dissect/target/plugins/os/windows/network.py b/dissect/target/plugins/os/windows/network.py index 8e268c017..d646b2496 100644 --- a/dissect/target/plugins/os/windows/network.py +++ b/dissect/target/plugins/os/windows/network.py @@ -13,7 +13,7 @@ ) from dissect.target.helpers.record import WindowsInterfaceRecord from dissect.target.helpers.regutil import RegistryKey -from dissect.target.plugins.general.network import NetworkPlugin +from dissect.target.plugins.os.default.network import NetworkPlugin from dissect.target.target import Target diff --git a/dissect/target/plugins/os/windows/registry.py b/dissect/target/plugins/os/windows/registry.py index e03707241..a76c13bb0 100644 --- a/dissect/target/plugins/os/windows/registry.py +++ b/dissect/target/plugins/os/windows/registry.py @@ -295,8 +295,7 @@ def subkey(self, key: str, subkey: str) -> KeyCollection: @internal def iterkeys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: warnings.warn("The iterkeys() function is deprecated, use keys() instead", DeprecationWarning) - for key in self.keys(keys): - yield key + yield from self.keys(keys) @internal def keys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: @@ -308,9 +307,7 @@ def keys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: for key in self._iter_controlset_keypaths(keys): try: - res = self.key(key) - for r in res: - yield r + yield from self.key(key) except RegistryKeyNotFoundError: pass except HiveUnavailableError: diff --git a/dissect/target/report.py b/dissect/target/report.py index 82afe04a3..ee33a7c93 100644 --- a/dissect/target/report.py +++ b/dissect/target/report.py @@ -1,11 +1,13 @@ +from __future__ import annotations + import argparse import dataclasses import textwrap from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Set, Type +from typing import Any from dissect.target import Target -from dissect.target.plugin import Plugin +from dissect.target.plugin import FunctionDescriptor, Plugin, PluginRegistry from dissect.target.target import Event BLOCK_INDENT = 4 * " " @@ -15,11 +17,11 @@ class TargetExecutionReport: target: Target - incompatible_plugins: Set[str] = dataclasses.field(default_factory=set) - registered_plugins: Set[str] = dataclasses.field(default_factory=set) + incompatible_plugins: set[str] = dataclasses.field(default_factory=set) + registered_plugins: set[str] = dataclasses.field(default_factory=set) - func_errors: Dict[str, str] = dataclasses.field(default_factory=dict) - func_execs: Set[str] = dataclasses.field(default_factory=set) + func_errors: dict[str, str] = dataclasses.field(default_factory=dict) + func_execs: set[str] = dataclasses.field(default_factory=set) def add_incompatible_plugin(self, plugin_name: str) -> None: self.incompatible_plugins.add(plugin_name) @@ -30,7 +32,7 @@ def add_registered_plugin(self, plugin_name: str) -> None: def add_func_error(self, func, stacktrace: str) -> None: self.func_errors[func] = stacktrace - def as_dict(self) -> Dict[str, Any]: + def as_dict(self) -> dict[str, Any]: return { "target": str(self.target), "incompatible_plugins": sorted(self.incompatible_plugins), @@ -42,19 +44,19 @@ def as_dict(self) -> Dict[str, Any]: @dataclass class ExecutionReport: - plugin_import_errors: Dict[str, str] = dataclasses.field(default_factory=dict) + plugin_import_errors: dict[str, str] = dataclasses.field(default_factory=dict) - target_reports: List[TargetExecutionReport] = dataclasses.field(default_factory=list) + target_reports: list[TargetExecutionReport] = dataclasses.field(default_factory=list) - cli_args: Dict[str, Any] = dataclasses.field(default_factory=dict) + cli_args: dict[str, Any] = dataclasses.field(default_factory=dict) def set_cli_args(self, args: argparse.Namespace) -> None: args = ((key, str(value)) for (key, value) in vars(args).items()) self.cli_args.update(args) - def set_plugin_stats(self, plugins: Dict[str, Any]) -> None: - for details in plugins.get("_failed", []): - self.plugin_import_errors[details["module"]] = "".join(details["stacktrace"]) + def set_plugin_stats(self, plugins: PluginRegistry) -> None: + for details in plugins.__failed__: + self.plugin_import_errors[details.module] = "".join(details.stacktrace) def get_formatted_report(self) -> str: blocks = [ @@ -76,15 +78,15 @@ def get_target_report(self, target: Target, create: bool = False) -> TargetExecu return target_report @staticmethod - def _get_plugin_name(plugin_cls): + def _get_plugin_name(plugin_cls) -> str: return f"{plugin_cls.__module__}.{plugin_cls.__qualname__}" def log_incompatible_plugin( self, target: Target, _, - plugin_cls: Optional[Type[Plugin]] = None, - plugin_desc: Optional[Dict[str, Any]] = None, + plugin_cls: type[Plugin] | None = None, + plugin_desc: FunctionDescriptor | None = None, ) -> None: if not plugin_cls and not plugin_desc: raise ValueError("Either `plugin_cls` or `plugin_desc` must be set") @@ -94,7 +96,7 @@ def log_incompatible_plugin( if plugin_cls: plugin_name = self._get_plugin_name(plugin_cls) elif plugin_desc: - plugin_name = plugin_desc["fullname"] + plugin_name = f"{plugin_desc.module}.{plugin_desc.qualname}" target_report.add_incompatible_plugin(plugin_name) @@ -112,7 +114,7 @@ def log_func_execution(self, target: Target, _, func: str) -> None: target_report = self.get_target_report(target, create=True) target_report.func_execs.add(func) - def set_event_callbacks(self, target_cls: Type[Target]) -> None: + def set_event_callbacks(self, target_cls: type[Target]) -> None: target_cls.set_event_callback( event_type=Event.INCOMPATIBLE_PLUGIN, event_callback=self.log_incompatible_plugin, @@ -130,7 +132,7 @@ def set_event_callbacks(self, target_cls: Type[Target]) -> None: event_callback=self.log_func_error, ) - def as_dict(self) -> Dict[str, Any]: + def as_dict(self) -> dict[str, Any]: return { "plugin_import_errors": self.plugin_import_errors, "target_reports": [report.as_dict() for report in self.target_reports], diff --git a/dissect/target/target.py b/dissect/target/target.py index 5dd6f81ec..a1bdc26f2 100644 --- a/dissect/target/target.py +++ b/dissect/target/target.py @@ -23,7 +23,7 @@ from dissect.target.helpers.loaderutil import extract_path_info from dissect.target.helpers.record import ChildTargetRecord from dissect.target.helpers.utils import StrEnum, parse_path_uri, slugify -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin log = logging.getLogger(__name__) @@ -118,6 +118,46 @@ def __init__(self, path: Union[str, Path] = None): self.fs = filesystem.RootFilesystem(self) + def __repr__(self) -> str: + return f"" + + def __getattr__(self, attr: str) -> Union[plugin.Plugin, Any]: + """Override of the default ``__getattr__`` so plugins and functions can be called from a ``Target`` object.""" + p, func = self.get_function(attr) + + if isinstance(func, property): + # If it's a property, execute it and return the result + try: + result = func.__get__(p) + self.send_event(Event.FUNC_EXEC, func=attr) + return result + except Exception: + if not attr.startswith("__"): + self.send_event( + Event.FUNC_EXEC_ERROR, + func=attr, + stacktrace=traceback.format_exc(), + ) + raise + + return func + + def __dir__(self) -> list[str]: + """Override the default ``__dir__`` to provide autocomplete for things like IPython.""" + funcs = [] + if self._os_plugin: + funcs = list(self._os_plugin.__functions__) + + for plugin_desc in plugin.plugins(self._os_plugin): + funcs.extend(plugin_desc.functions) + + result = set(self.__dict__.keys()) + result.update(self.__class__.__dict__.keys()) + result.update(object.__dict__.keys()) + result.update(funcs) + + return list(result) + @classmethod def set_event_callback(cls, *, event_type: Optional[Event] = None, event_callback: Callable) -> None: """Sets ``event_callbacks`` on a Target class. @@ -339,23 +379,23 @@ def _load_child_plugins(self) -> None: for plugin_desc in plugin.child_plugins(): try: - plugin_cls = plugin.load(plugin_desc) + plugin_cls: type[plugin.ChildTargetPlugin] = plugin.load(plugin_desc) child_plugin = plugin_cls(self) except PluginError: - self.log.exception("Failed to load child plugin: %s", plugin_desc["class"]) + self.log.exception("Failed to load child plugin: %s", plugin_desc.qualname) continue except Exception: - self.log.exception("Broken child plugin: %s", plugin_desc["class"]) + self.log.exception("Broken child plugin: %s", plugin_desc.qualname) continue try: child_plugin.check_compatible() self._child_plugins[child_plugin.__type__] = child_plugin except PluginError as e: - self.log.debug("Child plugin reported itself as incompatible: %s (%s)", plugin_desc["class"], e) + self.log.debug("Child plugin reported itself as incompatible: %s (%s)", plugin_desc.qualname, e) except Exception: self.log.exception( - "An exception occurred while checking for child plugin compatibility: %s", plugin_desc["class"] + "An exception occurred while checking for child plugin compatibility: %s", plugin_desc.qualname ) def open_child(self, child: Union[str, Path]) -> Target: @@ -452,7 +492,7 @@ def _init_os(self) -> None: if not len(self.disks) and not len(self.volumes) and not len(self.filesystems): raise TargetError(f"Failed to load target. No disks, volumes or filesystems: {self.path}") - candidates = [] + candidates: list[tuple[plugin.PluginDescriptor, type[plugin.OSPlugin], filesystem.Filesystem]] = [] for plugin_desc in plugin.os_plugins(): # Subclassed OS Plugins used to also subclass the detection of the @@ -466,25 +506,26 @@ def _init_os(self) -> None: # # Now subclassed OS Plugins are on the same detection "layer" as # regular OS Plugins, but can still inherit functions. - self.log.debug("Loading OS plugin: %s", plugin_desc["class"]) + qualname = plugin_desc.qualname + self.log.debug("Loading OS plugin: %s", qualname) try: - os_plugin = plugin.load(plugin_desc) + os_plugin: type[plugin.OSPlugin] = plugin.load(plugin_desc) fs = os_plugin.detect(self) except PluginError: - self.log.exception("Failed to load OS plugin: %s", plugin_desc["class"]) + self.log.exception("Failed to load OS plugin: %s", qualname) continue except Exception: - self.log.exception("Broken OS plugin: %s", plugin_desc["class"]) + self.log.exception("Broken OS plugin: %s", qualname) continue if not fs: continue - self.log.info("Found compatible OS plugin: %s", plugin_desc["class"]) + self.log.info("Found compatible OS plugin: %s", qualname) candidates.append((plugin_desc, os_plugin, fs)) fs = None - os_plugin = default.DefaultPlugin + os_plugin = DefaultPlugin if candidates: plugin_desc, os_plugin, fs = candidates[0] @@ -493,7 +534,7 @@ def _init_os(self) -> None: if len(candidate_plugin.mro()) > len(os_plugin.mro()): plugin_desc, os_plugin, fs = candidate_plugin_desc, candidate_plugin, candidate_fs - self.log.debug("Selected OS plugin: %s", plugin_desc["class"]) + self.log.debug("Selected OS plugin: %s", plugin_desc.qualname) else: # No OS detected self.log.warning("Failed to find OS plugin, falling back to default") @@ -517,7 +558,7 @@ def _mount_others(self) -> None: def add_plugin( self, - plugin_cls: Union[plugin.Plugin, type[plugin.Plugin]], + plugin_cls: plugin.Plugin | type[plugin.Plugin], check_compatible: bool = True, ) -> plugin.Plugin: """Add and register a plugin by class. @@ -564,6 +605,20 @@ def add_plugin( return p + def load_plugin(self, descriptor: plugin.PluginDescriptor | plugin.FunctionDescriptor) -> plugin.Plugin: + """Load a plugin by descriptor. + + Args: + plugin_desc: The descriptor of the plugin to load. + + Returns: + The loaded plugin instance. + + Raises: + PluginError: Raised when any exception occurs while trying to load the plugin. + """ + return self.add_plugin(plugin.load(descriptor)) + def _register_plugin_functions(self, plugin_inst: plugin.Plugin) -> None: """Internal function that registers all the exported functions from a given plugin. @@ -576,12 +631,15 @@ def _register_plugin_functions(self, plugin_inst: plugin.Plugin) -> None: if plugin_inst.__namespace__: self._functions[plugin_inst.__namespace__] = (plugin_inst, plugin_inst) + + for func in plugin_inst.__functions__: + self._functions[f"{plugin_inst.__namespace__}.{func}"] = (plugin_inst, None) else: for func in plugin_inst.__functions__: # If we getattr here, property members will be executed, so we do that in __getattr__ self._functions[func] = (plugin_inst, None) - def get_function(self, function: str) -> FunctionTuple: + def get_function(self, function: str | plugin.FunctionDescriptor) -> FunctionTuple: """Attempt to get a given function. If the function is not already registered, look for plugins that export the function and register them. @@ -596,42 +654,58 @@ def get_function(self, function: str) -> FunctionTuple: UnsupportedPluginError: Raised when plugins were found, but they were incompatible PluginError: Raised when any other exception occurs while trying to load the plugin. """ - if function not in self._functions: - causes = [] + if isinstance(function, plugin.FunctionDescriptor): + function_name = function.name - plugin_desc = None - for plugin_desc in plugin.lookup(function, self._os_plugin): + if function_name not in self._functions: try: - plugin_cls = plugin.load(plugin_desc) - self.add_plugin(plugin_cls) - self.log.debug("Found compatible plugin '%s' for function '%s'", plugin_desc["class"], function) - break - except UnsupportedPluginError as e: - self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=plugin_desc) - causes.append(e) - else: - if plugin_desc: - # In this case we made at least one iteration but it was skipped due incompatibility. - # Just take the last known cause for now + self.load_plugin(function) + except UnsupportedPluginError: + self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=function) raise UnsupportedPluginError( - f"Unsupported function `{function}` for target with OS plugin {self._os_plugin}", - extra=causes[1:] if len(causes) > 1 else None, - ) from causes[0] if causes else None + f"Unsupported function `{function.name}` (`{function.module}`)", + ) + + else: + function_name = function + + if function not in self._functions: + causes = [] + + descriptor = None + for descriptor in plugin.lookup(function, self._os_plugin): + try: + self.load_plugin(descriptor) + self.log.debug("Found compatible plugin '%s' for function '%s'", descriptor.qualname, function) + break + except UnsupportedPluginError as e: + self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=descriptor) + causes.append(e) + else: + if descriptor: + # In this case we made at least one iteration but it was skipped due incompatibility. + # Just take the last known cause for now + raise UnsupportedPluginError( + f"Unsupported function `{function}` for target with OS plugin {self._os_plugin}", + extra=causes[1:] if len(causes) > 1 else None, + ) from causes[0] if causes else None # We still ended up with no compatible plugins - if function not in self._functions: - raise PluginNotFoundError(f"Can't find plugin with function `{function}`") + if function_name not in self._functions: + raise PluginNotFoundError(f"Can't find plugin with function `{function_name}`") - p, func = self._functions[function] + p, func = self._functions[function_name] if func is None: - func = getattr(p.__class__, function) + method_attr = function_name.rpartition(".")[2] if p.__namespace__ else function_name + func = getattr(p.__class__, method_attr) + if not isinstance(func, property) or ( isinstance(func, property) and getattr(func.fget, "__persist__", False) ): # If the persist flag is set on a property, store the property result in the function cache # This is so we don't have to evaluate the property again - func = getattr(p, function) - self._functions[function] = (p, func) + func = getattr(p, method_attr) + self._functions[function_name] = (p, func) return p, func @@ -650,46 +724,6 @@ def has_function(self, function: str) -> bool: except PluginError: return False - def __getattr__(self, attr: str) -> Union[plugin.Plugin, Any]: - """Override of the default __getattr__ so plugins and functions can be called from a ``Target`` object.""" - p, func = self.get_function(attr) - - if isinstance(func, property): - # If it's a property, execute it and return the result - try: - result = func.__get__(p) - self.send_event(Event.FUNC_EXEC, func=attr) - return result - except Exception: - if not attr.startswith("__"): - self.send_event( - Event.FUNC_EXEC_ERROR, - func=attr, - stacktrace=traceback.format_exc(), - ) - raise - - return func - - def __dir__(self): - """Override the default __dir__ to provide autocomplete for things like IPython.""" - funcs = [] - if self._os_plugin: - funcs = list(self._os_plugin.__functions__) - - for plugin_desc in plugin.plugins(self._os_plugin): - funcs.extend(plugin_desc["functions"]) - - result = set(self.__dict__.keys()) - result.update(self.__class__.__dict__.keys()) - result.update(object.__dict__.keys()) - result.update(funcs) - - return list(result) - - def __repr__(self): - return f"" - T = TypeVar("T") diff --git a/dissect/target/tools/build_pluginlist.py b/dissect/target/tools/build_pluginlist.py index 87b9fc18f..aadc1208e 100644 --- a/dissect/target/tools/build_pluginlist.py +++ b/dissect/target/tools/build_pluginlist.py @@ -3,12 +3,12 @@ import argparse import logging -import pprint +import textwrap from dissect.target import plugin -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity") args = parser.parse_args() @@ -25,7 +25,19 @@ def main(): logging.basicConfig(level=logging.CRITICAL) pluginlist = plugin.generate() - print(f"PLUGINS = \\\n{pprint.pformat(pluginlist)}") + template = """ + from dissect.target.plugin import ( + FailureDescriptor, + FunctionDescriptor, + FunctionDescriptorLookup, + PluginDescriptor, + PluginDescriptorLookup, + PluginRegistry, + ) + + PLUGINS = {} + """ + print(textwrap.dedent(template).format(pluginlist)) if __name__ == "__main__": diff --git a/dissect/target/tools/dump/run.py b/dissect/target/tools/dump/run.py index a4c9245ac..aedce369d 100644 --- a/dissect/target/tools/dump/run.py +++ b/dissect/target/tools/dump/run.py @@ -27,7 +27,7 @@ cached_sink_writers, ) from dissect.target.tools.utils import ( - PluginFunction, + FunctionDescriptor, configure_generic_arguments, execute_function_on_target, find_and_filter_plugins, @@ -52,7 +52,7 @@ def get_targets(targets: list[str]) -> Iterator[Target]: yield target -def execute_function(target: Target, function: PluginFunction) -> TargetRecordDescriptor: +def execute_function(target: Target, function: FunctionDescriptor) -> TargetRecordDescriptor: """ Execute function `function` on provided target `target` and return a generator with the records produced. @@ -91,7 +91,7 @@ def produce_target_func_pairs( targets: Iterable[Target], functions: str, state: DumpState, -) -> Iterator[tuple[Target, PluginFunction]]: +) -> Iterator[tuple[Target, FunctionDescriptor]]: """ Return a generator with target and function pairs for execution. @@ -102,7 +102,7 @@ def produce_target_func_pairs( pairs_to_skip.update((str(sink.target_path), sink.func) for sink in state.finished_sinks) for target in targets: - for func_def in find_and_filter_plugins(target, functions): + for func_def in find_and_filter_plugins(functions, target): if state and (target.path, func_def.name) in pairs_to_skip: log.info( "Skipping target/func pair since its marked as done in provided state", diff --git a/dissect/target/tools/dump/utils.py b/dissect/target/tools/dump/utils.py index 3c2a6dcb5..422d07590 100644 --- a/dissect/target/tools/dump/utils.py +++ b/dissect/target/tools/dump/utils.py @@ -32,7 +32,7 @@ from flow.record.jsonpacker import JsonRecordPacker from dissect.target import Target -from dissect.target.plugin import PluginFunction +from dissect.target.plugin import FunctionDescriptor log = structlog.get_logger(__name__) @@ -74,13 +74,13 @@ def get_nested_attr(obj: Any, nested_attr: str) -> Any: @lru_cache(maxsize=DEST_DIR_CACHE_SIZE) -def get_sink_dir_by_target(target: Target, function: PluginFunction) -> Path: +def get_sink_dir_by_target(target: Target, function: FunctionDescriptor) -> Path: func_first_name, _, _ = function.name.partition(".") return Path(target.name) / func_first_name @functools.lru_cache(maxsize=DEST_DIR_CACHE_SIZE) -def get_sink_dir_by_func(target: Target, function: PluginFunction) -> Path: +def get_sink_dir_by_func(target: Target, function: FunctionDescriptor) -> Path: func_first_name, _, _ = function.name.partition(".") return Path(func_first_name) / target.name diff --git a/dissect/target/tools/query.py b/dissect/target/tools/query.py index 5ab83560e..897e3d0ad 100644 --- a/dissect/target/tools/query.py +++ b/dissect/target/tools/query.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import annotations + import argparse import logging import pathlib @@ -9,8 +11,9 @@ from typing import Callable from flow.record import RecordPrinter, RecordStreamWriter, RecordWriter +from flow.record.base import AbstractWriter -from dissect.target import Target +from dissect.target import Target, plugin from dissect.target.exceptions import ( FatalError, PluginNotFoundError, @@ -18,7 +21,12 @@ UnsupportedPluginError, ) from dissect.target.helpers import cache, record_modifier -from dissect.target.plugin import PLUGINS, OSPlugin, Plugin, find_plugin_functions +from dissect.target.plugin import PLUGINS, OSPlugin, Plugin, find_functions +from dissect.target.plugins.general.plugins import ( + _get_default_functions, + generate_functions_json, + generate_functions_overview, +) from dissect.target.report import ExecutionReport from dissect.target.tools.utils import ( args_to_uri, @@ -41,7 +49,7 @@ USAGE_FORMAT_TMPL = "{prog} -f {name}{usage}" -def record_output(strings=False, json=False): +def record_output(strings: bool = False, json: bool = False) -> AbstractWriter: if json: return RecordWriter("jsonfile://-") @@ -53,8 +61,51 @@ def record_output(strings=False, json=False): return RecordStreamWriter(fp) +def list_plugins( + targets: list[str] | None = None, + patterns: str = "", + include_children: bool = False, + as_json: bool = False, + argv: list[str] | None = None, +) -> None: + collected = set() + if targets: + for target in Target.open_all(targets, include_children): + funcs, _ = find_functions(patterns, target, compatibility=True, show_hidden=True) + collected.update(funcs) + elif patterns: + funcs, _ = find_functions(patterns, Target(), show_hidden=True) + collected.update(funcs) + else: + collected.update(_get_default_functions()) + + target = Target() + fparser = generate_argparse_for_bound_method(target.plugins, usage_tmpl=USAGE_FORMAT_TMPL) + fargs, rest = fparser.parse_known_args(argv or []) + + # Display in a user friendly manner + if collected: + if as_json: + print('{"plugins": ', end="") + print(generate_functions_json(collected), end="") + else: + print(generate_functions_overview(collected, include_docs=fargs.print_docs)) + + # No real targets specified, show the available loaders + if not targets: + fparser = generate_argparse_for_bound_method(target.loaders, usage_tmpl=USAGE_FORMAT_TMPL) + fargs, rest = fparser.parse_known_args(rest) + del fargs.as_json + if as_json: + print(', "loaders": ', end="") + target.loaders(**vars(fargs), as_json=as_json) + + if as_json: + print("}") + + @catch_sigpipe -def main(): +def main() -> None: help_formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser( description="dissect.target", @@ -78,7 +129,7 @@ def main(): "--list", action="store", nargs="?", - const="*", + const="", default=None, help="list (matching) available plugins and loaders", ) @@ -106,8 +157,7 @@ def main(): "--rewrite-cache", action="store_true", help=( - "force cache files to be rewritten (has no effect if either --no-cache " - "or --only-read-cache are specified)" + "force cache files to be rewritten (has no effect if either --no-cache or --only-read-cache are specified)" ), ) parser.add_argument("--cmdb", action="store_true") @@ -149,14 +199,17 @@ def main(): # Show help for a function or in general if "-h" in rest or "--help" in rest: - found_functions, _ = find_plugin_functions(None, args.function, compatibility=False) + found_functions, _ = find_functions(args.function) if not len(found_functions): parser.error("function(s) not found, see -l for available plugins") + func = found_functions[0] - if issubclass(func.class_object, OSPlugin): + plugin_class = plugin.load(func) + if issubclass(plugin_class, OSPlugin): obj = getattr(OSPlugin, func.method_name) else: - obj = getattr(func.class_object, func.method_name) + obj = getattr(plugin_class, func.method_name) + if isinstance(obj, type) and issubclass(obj, Plugin): parser = generate_argparse_for_plugin_class(obj, usage_tmpl=USAGE_FORMAT_TMPL) elif isinstance(obj, Callable) or isinstance(obj, property): @@ -168,41 +221,8 @@ def main(): # Show the list of available plugins for the given optional target and optional # search pattern, only display plugins that can be applied to ANY targets - if args.list: - collected_plugins = [] - - if targets: - for plugin_target in Target.open_all(targets, args.children): - funcs, _ = find_plugin_functions(plugin_target, args.list, compatibility=True, show_hidden=True) - for func in funcs: - collected_plugins.append(func) - else: - funcs, _ = find_plugin_functions(Target(), args.list, compatibility=False, show_hidden=True) - for func in funcs: - collected_plugins.append(func) - - target = Target() - fparser = generate_argparse_for_bound_method(target.plugins, usage_tmpl=USAGE_FORMAT_TMPL) - fargs, rest = fparser.parse_known_args(rest) - - # Display in a user friendly manner - if collected_plugins: - if args.json: - print('{"plugins": ', end="") - target.plugins(collected_plugins, as_json=args.json) - - # No real targets specified, show the available loaders - if not targets: - fparser = generate_argparse_for_bound_method(target.loaders, usage_tmpl=USAGE_FORMAT_TMPL) - fargs, rest = fparser.parse_known_args(rest) - del fargs.as_json - if args.json: - print(', "loaders": ', end="") - target.loaders(**vars(fargs), as_json=args.json) - - if args.json: - print("}") - + if args.list is not None: + list_plugins(targets, args.list, args.children, args.json, rest) parser.exit() if not targets: @@ -211,6 +231,19 @@ def main(): if not args.function: parser.error("argument -f/--function is required") + if args.report_dir and not args.report_dir.is_dir(): + parser.error(f"--report-dir {args.report_dir} is not a valid directory") + + funcs, invalid_funcs = find_functions(args.function) + if any(invalid_funcs): + parser.error(f"argument -f/--function contains invalid plugin(s): {', '.join(invalid_funcs)}") + + excluded_funcs, invalid_excluded_funcs = find_functions(args.excluded_functions) + if any(invalid_excluded_funcs): + parser.error( + f"argument -xf/--excluded-functions contains invalid plugin(s): {', '.join(invalid_excluded_funcs)}", + ) + # Verify uniformity of output types, otherwise default to records. # Note that this is a heuristic, the targets are not opened yet because of # performance, so it might generate a false positive @@ -224,28 +257,12 @@ def main(): # The only scenario that might cause this is with # custom plugins with idiosyncratic output across OS-versions/branches. output_types = set() - funcs, invalid_funcs = find_plugin_functions(None, args.function, compatibility=False) - - if any(invalid_funcs): - parser.error(f"argument -f/--function contains invalid plugin(s): {', '.join(invalid_funcs)}") - - excluded_funcs, invalid_excluded_funcs = find_plugin_functions( - None, - args.excluded_functions, - compatibility=False, - ) - - if any(invalid_excluded_funcs): - parser.error( - f"argument -xf/--excluded-functions contains invalid plugin(s): {', '.join(invalid_excluded_funcs)}", - ) - excluded_func_paths = {excluded_func.path for excluded_func in excluded_funcs} for func in funcs: if func.path in excluded_func_paths: continue - output_types.add(func.output_type) + output_types.add(func.output) default_output_type = None @@ -254,9 +271,6 @@ def main(): log.warning("Mixed output types detected: %s. Only outputting records.", ",".join(output_types)) default_output_type = "record" - if args.report_dir and not args.report_dir.is_dir(): - parser.error(f"--report-dir {args.report_dir} is not a valid directory") - execution_report = ExecutionReport() execution_report.set_cli_args(args) execution_report.set_event_callbacks(Target) @@ -277,18 +291,14 @@ def main(): yield_entries = [] first_seen_output_type = default_output_type - cli_params_unparsed = rest - - excluded_funcs, _ = find_plugin_functions(target, args.excluded_functions, compatibility=False) - excluded_func_paths = {excluded_func.path for excluded_func in excluded_funcs} - for func_def in find_and_filter_plugins(target, args.function, excluded_func_paths): + for func_def in find_and_filter_plugins(args.function, target, excluded_func_paths): # If the default type is record (meaning we skip everything else) # and actual output type is not record, continue. # We perform this check here because plugins that require output files/dirs # will exit if we attempt to exec them without (because they are implied by the wildcard). # Also this saves cycles of course. - if default_output_type == "record" and func_def.output_type != "record": + if default_output_type == "record" and func_def.output != "record": continue if args.dry_run: @@ -296,9 +306,7 @@ def main(): continue try: - output_type, result, cli_params_unparsed = execute_function_on_target( - target, func_def, cli_params_unparsed - ) + output_type, result, rest = execute_function_on_target(target, func_def, rest) except UnsupportedPluginError as e: target.log.error( "Unsupported plugin for %s: %s", @@ -315,7 +323,10 @@ def main(): fatal.emit_last_message(target.log.error) parser.exit(1) except Exception: - target.log.error("Exception while executing function `%s`", func_def, exc_info=True) + target.log.error( + "Exception while executing function `%s` (`%s`)", func_def.name, func_def.path, exc_info=True + ) + target.log.debug("Function info: %s", func_def) continue if first_seen_output_type and output_type != first_seen_output_type: diff --git a/dissect/target/tools/shell.py b/dissect/target/tools/shell.py index 7334f2c81..e7c3aae92 100644 --- a/dissect/target/tools/shell.py +++ b/dissect/target/tools/shell.py @@ -33,7 +33,7 @@ ) from dissect.target.filesystem import FilesystemEntry from dissect.target.helpers import cyber, fsutil, regutil -from dissect.target.plugin import PluginFunction, alias, arg, clone_alias +from dissect.target.plugin import FunctionDescriptor, alias, arg, clone_alias from dissect.target.target import Target from dissect.target.tools.fsutils import ( fmt_ls_colors, @@ -396,13 +396,13 @@ def _handle_command(self, line: str) -> bool | None: # execution command, command_args_str, line = self.parseline(line) - if plugins := list(find_and_filter_plugins(self.target, command, [])): - return self._exec_target(plugins, command_args_str) + if functions := list(find_and_filter_plugins(command, self.target)): + return self._exec_target(functions, command_args_str) # We didn't execute a function on the target return None - def _exec_target(self, funcs: list[PluginFunction], command_args_str: str) -> bool: + def _exec_target(self, funcs: list[FunctionDescriptor], command_args_str: str) -> bool: """Command exection helper for target plugins.""" def _exec_(argparts: list[str], stdout: TextIO) -> None: diff --git a/dissect/target/tools/utils.py b/dissect/target/tools/utils.py index 8dcac009a..56266ec69 100644 --- a/dissect/target/tools/utils.py +++ b/dissect/target/tools/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import errno import inspect @@ -10,18 +12,16 @@ from functools import wraps from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Type, Union +from typing import Any, Callable, Iterator from dissect.target import Target -from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers import docs, keychain from dissect.target.helpers.docs import get_docstring from dissect.target.loader import LOADERS_BY_SCHEME from dissect.target.plugin import ( - OSPlugin, + FunctionDescriptor, Plugin, - PluginFunction, - find_plugin_functions, + find_functions, get_external_module_paths, load_modules_from_paths, ) @@ -65,9 +65,9 @@ def process_generic_arguments(args: argparse.Namespace) -> None: def generate_argparse_for_bound_method( method: Callable, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a bound `Plugin` class method""" + """Generate an ``argparse.ArgumentParser`` for a bound ``Plugin` class method.""" # allow functools.partial wrapped method while hasattr(method, "func"): @@ -82,9 +82,9 @@ def generate_argparse_for_bound_method( def generate_argparse_for_unbound_method( method: Callable, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for an unbound `Plugin` class method""" + """Generate an ``argparse.ArgumentParser`` for an unbound ``Plugin`` class method.""" if not inspect.isfunction(method): raise ValueError(f"Value `{method}` is not an unbound plugin method") @@ -124,10 +124,10 @@ def generate_argparse_for_unbound_method( def generate_argparse_for_plugin_class( - plugin_cls: Type[Plugin], - usage_tmpl: Optional[str] = None, + plugin_cls: type[Plugin], + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a `Plugin` class""" + """Generate an ``argparse.ArgumentParser`` for a ``Plugin`` class.""" if not isinstance(plugin_cls, type) or not issubclass(plugin_cls, Plugin): raise ValueError(f"`plugin_cls` must be a valid plugin class, not `{plugin_cls}`") @@ -149,9 +149,9 @@ def generate_argparse_for_plugin_class( def generate_argparse_for_plugin( plugin_instance: Plugin, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a `Plugin` instance""" + """Generate an ``argparse.ArgumentParser`` for a ``Plugin`` instance.""" if not isinstance(plugin_instance, Plugin): raise ValueError(f"`plugin_instance` must be a valid plugin instance, not `{plugin_instance}`") @@ -162,84 +162,35 @@ def generate_argparse_for_plugin( return generate_argparse_for_plugin_class(plugin_instance.__class__, usage_tmpl=usage_tmpl) -def plugin_factory( - target: Target, plugin: Union[type, object], funcname: str, namespace: Optional[str] -) -> tuple[Plugin, str]: - if hasattr(target._loader, "instance"): - return target.get_function(funcname, namespace=namespace) - - if isinstance(plugin, type): - plugin_obj = plugin(target) - target_attr = getattr(plugin_obj, funcname) - return plugin_obj, target_attr - else: - return plugin, getattr(plugin, funcname) - - def execute_function_on_target( target: Target, - func: PluginFunction, - cli_params: Optional[List[str]] = None, -) -> Tuple[str, Any, List[str]]: - """ - Execute function `func` on provided target `target` with provided `cli_params` list. - """ + func: FunctionDescriptor, + arguments: list[str] | None = None, +) -> tuple[str, Any, list[str]]: + """Execute function on provided target with provided arguments.""" - cli_params = cli_params or [] + arguments = arguments or [] - target_attr = get_target_attribute(target, func) - plugin_method, parser = plugin_function_with_argparser(target_attr) + func_cls, func_obj = target.get_function(func.name) + plugin_method, parser = plugin_function_with_argparser(func_obj) if parser: - parsed_params, cli_params = parser.parse_known_args(cli_params) - method_kwargs = vars(parsed_params) - value = plugin_method(**method_kwargs) + known_args, rest = parser.parse_known_args(arguments) + value = plugin_method(**vars(known_args)) + elif isinstance(func_obj, property): + rest = arguments + value = func_obj.__get__(func_cls) else: - value = target_attr + rest = arguments + value = func_obj output_type = getattr(plugin_method, "__output__", "default") if plugin_method else "default" - return (output_type, value, cli_params) - - -def get_target_attribute(target: Target, func: PluginFunction) -> Union[Plugin, Callable]: - """Retrieves the function attribute from the target. - - If the function does not exist yet, it will attempt to load it into the target. - - Args: - target: The target we wish to run the function on. - func: The function to run on the target. - - Returns: - The function, either plugin or a callable to execute. - - Raises: - UnsupportedPluginError: When the function was incompatible with the target. - """ - plugin_class = func.class_object - if ns := getattr(func, "plugin_desc", {}).get("namespace", None): - plugin_class = getattr(target, ns) - elif target.has_function(func.method_name): - # If the function is already attached, use the one inside the target. - plugin_class, _ = target.get_function(func.method_name) - elif issubclass(plugin_class, OSPlugin): - # OS plugin does not need to be added - plugin_class = target._os_plugin - else: - try: - target.add_plugin(plugin_class) - except UnsupportedPluginError as e: - raise UnsupportedPluginError( - f"Unsupported function `{func.method_name}` for target with plugin {func.class_object}" - ) from e - - _, target_attr = plugin_factory(target, plugin_class, func.method_name, func.plugin_desc["namespace"]) - return target_attr + return (output_type, value, rest) def plugin_function_with_argparser( - target_attr: Union[Plugin, Callable], -) -> tuple[Optional[Iterator], Optional[argparse.ArgumentParser]]: + target_attr: Plugin | Callable, +) -> tuple[Callable | None, argparse.ArgumentParser | None]: """Resolves which plugin function to execute, and creates the argument parser for said plugin.""" plugin_method = None parser = None @@ -251,7 +202,7 @@ def plugin_function_with_argparser( if not plugin_obj.__namespace__: raise ValueError(f"Plugin {plugin_obj} is not callable") - plugin_method = plugin_obj.get_all_records + plugin_method = plugin_obj.__call__ parser = generate_argparse_for_plugin(plugin_obj) elif callable(target_attr): plugin_method = target_attr @@ -259,7 +210,7 @@ def plugin_function_with_argparser( return plugin_method, parser -def persist_execution_report(output_dir: Path, report_data: Dict, timestamp: datetime) -> Path: +def persist_execution_report(output_dir: Path, report_data: dict, timestamp: datetime) -> Path: timestamp = timestamp.strftime("%Y-%m-%d-%H%M%S") report_filename = f"target-report-{timestamp}.json" report_full_path = output_dir / report_filename @@ -268,7 +219,7 @@ def persist_execution_report(output_dir: Path, report_data: Dict, timestamp: dat def catch_sigpipe(func: Callable) -> Callable: - """Catches KeyboardInterrupt and BrokenPipeError (OSError 22 on Windows).""" + """Catches ``KeyboardInterrupt`` and ``BrokenPipeError`` (``OSError 22`` on Windows).""" @wraps(func) def wrapper(*args, **kwargs): @@ -314,13 +265,13 @@ def args_to_uri(targets: list[str], loader_name: str, rest: list[str]) -> list[s def find_and_filter_plugins( - target: Target, functions: str, excluded_func_paths: set[str] = None -) -> Iterator[PluginFunction]: + functions: str, target: Target, excluded_func_paths: set[str] | None = None +) -> Iterator[FunctionDescriptor]: # Keep a set of plugins that were already executed on the target. executed_plugins = set() excluded_func_paths = excluded_func_paths or set() - func_defs, _ = find_plugin_functions(target, functions, compatibility=False) + func_defs, _ = find_functions(functions, target) for func_def in func_defs: if func_def.path in excluded_func_paths: diff --git a/tests/_data/registration/plugin.py b/tests/_data/registration/plugin.py index aa5e732d6..1d44a6693 100644 --- a/tests/_data/registration/plugin.py +++ b/tests/_data/registration/plugin.py @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2f7dd22e5c0b8b50bfed941d8f1829bddf54ce682d947a0b75e7469c074126d7 -size 272 +oid sha256:81267a96b9b1c05025328fc60cbb71bdbc0563390c0dec12ee287c6e7ffdc1c6 +size 298 diff --git a/tests/conftest.py b/tests/conftest.py index 6184dd70e..546252293 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.regutil import VirtualHive, VirtualKey, VirtualValue from dissect.target.plugin import OSPlugin -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.plugins.os.unix.bsd.citrix._os import CitrixPlugin from dissect.target.plugins.os.unix.bsd.osx._os import MacPlugin @@ -261,7 +261,7 @@ def target_bare(tmp_path: pathlib.Path) -> Iterator[Target]: @pytest.fixture def target_default(tmp_path: pathlib.Path) -> Iterator[Target]: - yield make_os_target(tmp_path, default.DefaultPlugin) + yield make_os_target(tmp_path, DefaultPlugin) @pytest.fixture diff --git a/tests/helpers/test_docs.py b/tests/helpers/test_docs.py index 4b50afe8f..ba104a024 100644 --- a/tests/helpers/test_docs.py +++ b/tests/helpers/test_docs.py @@ -21,7 +21,7 @@ def test_docs_plugin_functions_desc() -> None: assert functions_short_desc desc_lines = functions_short_desc.splitlines() - assert len(desc_lines) == 3 + assert len(desc_lines) == 2 assert "iis.logs" in functions_short_desc assert "Return contents of IIS (v7 and above) log files." in functions_short_desc assert "output: records" in functions_short_desc diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index 152743832..825c2c539 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -158,8 +158,7 @@ def test_logrotate(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file("var/log/apache2/access.log.2", data_file) fs_unix.map_file("var/log/apache2/access.log.3", data_file) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() assert len(access_log_paths) == 4 assert len(error_log_paths) == 0 @@ -172,8 +171,7 @@ def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file_fh("custom/log/location/access.log.2", BytesIO(b"Foo2")) fs_unix.map_file_fh("custom/log/location/access.log.3", BytesIO(b"Foo3")) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() assert len(access_log_paths) == 4 assert len(error_log_paths) == 0 @@ -192,9 +190,8 @@ def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) fs_unix.map_file_fh("custom/log/location/old.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/old_error.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/new_error.log", BytesIO(b"New")) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() # Log paths are returned in alphabetical order assert str(access_log_paths[0]) == "/custom/log/location/new.log" diff --git a/tests/plugins/apps/webserver/test_caddy.py b/tests/plugins/apps/webserver/test_caddy.py index 02f3b7439..91e6f04b8 100644 --- a/tests/plugins/apps/webserver/test_caddy.py +++ b/tests/plugins/apps/webserver/test_caddy.py @@ -4,11 +4,13 @@ from flow.record.fieldtypes import datetime as dt +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.apps.webserver.caddy import CaddyPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_plugins_apps_webservers_caddy_txt(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_txt(target_unix: Target, fs_unix: VirtualFilesystem) -> None: tz = timezone(timedelta(hours=-7)) fs_unix.map_file_fh( "var/log/caddy_access.log", @@ -30,7 +32,7 @@ def test_plugins_apps_webservers_caddy_txt(target_unix, fs_unix): assert record.bytes_sent == 2326 -def test_plugins_apps_webservers_caddy_json(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_json(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file( "var/log/caddy_access.log", absolute_path("_data/plugins/apps/webserver/caddy/access.log"), @@ -51,7 +53,7 @@ def test_plugins_apps_webservers_caddy_json(target_unix, fs_unix): assert record.bytes_sent == 12 -def test_plugins_apps_webservers_caddy_config(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/caddy/Caddyfile") fs_unix.map_file("etc/caddy/Caddyfile", config_file) @@ -59,15 +61,14 @@ def test_plugins_apps_webservers_caddy_config(target_unix, fs_unix): fs_unix.map_file_fh("var/www/log/access.log", BytesIO(b"Foo")) fs_unix.map_file_fh("var/log/caddy/access.log", BytesIO(b"Foo")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 2 assert str(log_paths[0]) == "/var/log/caddy/access.log" assert str(log_paths[1]) == "/var/www/log/access.log" -def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/caddy/Caddyfile") fs_unix.map_file("etc/caddy/Caddyfile", config_file) @@ -76,13 +77,12 @@ def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix, fs_un fs_unix.map_file_fh("var/www/log/access.log.2", BytesIO(b"Foo2")) fs_unix.map_file_fh("var/www/log/access.log.3", BytesIO(b"Foo3")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 4 -def test_plugins_apps_webservers_caddy_config_commented(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config_commented(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ root /var/www/html 1.example.com { @@ -102,8 +102,7 @@ def test_plugins_apps_webservers_caddy_config_commented(target_unix, fs_unix): fs_unix.map_file_fh("var/www/log/new.log", BytesIO(b"Foo")) fs_unix.map_file_fh("completely/disabled/access.log", BytesIO(b"Foo")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 3 assert str(log_paths[0]) == "/var/www/log/old.log" diff --git a/tests/plugins/apps/webserver/test_citrix.py b/tests/plugins/apps/webserver/test_citrix.py index 8c115f8dd..32f856742 100644 --- a/tests/plugins/apps/webserver/test_citrix.py +++ b/tests/plugins/apps/webserver/test_citrix.py @@ -75,8 +75,7 @@ def test_error_logs(target_citrix: Target, fs_bsd: VirtualFilesystem) -> None: fs_bsd.map_file("var/log/httperror-vpn.log", BytesIO(b"Foo")) fs_bsd.map_file("var/log/httperror.log", BytesIO(b"Bar")) - target_citrix.add_plugin(CitrixWebserverPlugin) - access_log_paths, error_log_paths = target_citrix.citrix.get_log_paths() + access_log_paths, error_log_paths = CitrixWebserverPlugin(target_citrix).get_log_paths() assert len(error_log_paths) == 2 diff --git a/tests/plugins/apps/webserver/test_nginx.py b/tests/plugins/apps/webserver/test_nginx.py index 77337e15e..646fb2648 100644 --- a/tests/plugins/apps/webserver/test_nginx.py +++ b/tests/plugins/apps/webserver/test_nginx.py @@ -2,11 +2,13 @@ from datetime import datetime, timezone from io import BytesIO +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.apps.webserver.nginx import NginxPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_plugins_apps_webservers_nginx_txt(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_txt(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log") fs_unix.map_file("var/log/nginx/access.log", data_file) @@ -26,7 +28,7 @@ def test_plugins_apps_webservers_nginx_txt(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_ipv6(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_ipv6(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log") fs_unix.map_file("var/log/nginx/access.log", data_file) @@ -45,7 +47,7 @@ def test_plugins_apps_webservers_nginx_ipv6(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_gz(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_gz(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log.gz") fs_unix.map_file("var/log/nginx/access.log.1.gz", data_file) @@ -64,7 +66,7 @@ def test_plugins_apps_webservers_nginx_gz(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_bz2(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_bz2(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log.bz2") fs_unix.map_file("var/log/nginx/access.log.1.bz2", data_file) @@ -83,20 +85,19 @@ def test_plugins_apps_webservers_nginx_bz2(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_config(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/nginx/nginx.conf") fs_unix.map_file("etc/nginx/nginx.conf", config_file) for i, log in enumerate(["access.log", "domain1.access.log", "domain2.access.log", "big.server.access.log"]): fs_unix.map_file_fh(f"opt/logs/{i}/{log}", BytesIO(b"Foo")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert len(log_paths) == 4 -def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/nginx/nginx.conf") fs_unix.map_file("etc/nginx/nginx.conf", config_file) fs_unix.map_file_fh("opt/logs/0/access.log", BytesIO(b"Foo1")) @@ -105,13 +106,12 @@ def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix, fs_un fs_unix.map_file_fh("opt/logs/1/domain1.access.log", BytesIO(b"Foo4")) fs_unix.map_file_fh("var/log/nginx/access.log", BytesIO(b"Foo5")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert len(log_paths) == 5 -def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ # access_log /foo/bar/old.log main; access_log /foo/bar/new.log main; @@ -119,8 +119,7 @@ def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix, fs_uni fs_unix.map_file_fh("etc/nginx/nginx.conf", BytesIO(textwrap.dedent(config).encode())) fs_unix.map_file_fh("foo/bar/new.log", BytesIO(b"New")) fs_unix.map_file_fh("foo/bar/old.log", BytesIO(b"Old")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert str(log_paths[0]) == "/foo/bar/old.log" assert str(log_paths[1]) == "/foo/bar/new.log" diff --git a/tests/plugins/general/test_default.py b/tests/plugins/general/test_default.py index 6977192b2..179b84d8d 100644 --- a/tests/plugins/general/test_default.py +++ b/tests/plugins/general/test_default.py @@ -2,7 +2,7 @@ import pytest -from dissect.target.plugins.general.default import DefaultPlugin +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.target import Target diff --git a/tests/plugins/general/test_network.py b/tests/plugins/general/test_network.py index 3819a3003..dd9abd6f8 100644 --- a/tests/plugins/general/test_network.py +++ b/tests/plugins/general/test_network.py @@ -7,7 +7,7 @@ UnixInterfaceRecord, WindowsInterfaceRecord, ) -from dissect.target.plugins.general.network import InterfaceRecord, NetworkPlugin +from dissect.target.plugins.os.default.network import InterfaceRecord, NetworkPlugin from dissect.target.target import Target diff --git a/tests/plugins/general/test_plugins.py b/tests/plugins/general/test_plugins.py index c2745cec1..e5ab51c64 100644 --- a/tests/plugins/general/test_plugins.py +++ b/tests/plugins/general/test_plugins.py @@ -1,41 +1,22 @@ -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, patch -import dissect.target.plugins.general.plugins as plugin +from dissect.target import plugin from dissect.target.plugins.general.plugins import ( PluginListPlugin, - categorize_plugins, - dictify_module_recursive, - output_plugin_description_recursive, - update_dict_recursive, + _categorize_functions, + _generate_plugin_tree_overview, ) -def test_dictify_module(): - last_value = Mock() - - output_dict = dictify_module_recursive(["hello", "world"], last_value) - - assert output_dict == {"hello": {"world": last_value}} - - -def test_update_dict(): - tmp_dictionary = dict() - - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world"], None)) - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "lawrence"], None)) - - assert tmp_dictionary == {"hello": {"world": None, "lawrence": None}} - - -def test_plugin_description(): - description = [x for x in output_plugin_description_recursive(PluginListPlugin, False)] +def test_plugin_description() -> None: + description = [x for x in _generate_plugin_tree_overview(PluginListPlugin, False)] assert description == ["plugins - Print all available plugins. (output: no output)"] -def test_plugin_description_compacting(): - module = dictify_module_recursive(["hello", "world"], PluginListPlugin) +def test_plugin_description_compacting() -> None: + module = {"hello": {"world": PluginListPlugin}} - description = [x for x in output_plugin_description_recursive(module, False)] + description = [x for x in _generate_plugin_tree_overview(module, False)] assert description == [ "hello:", " world:", @@ -43,13 +24,10 @@ def test_plugin_description_compacting(): ] -def test_plugin_description_in_dict_multiple(): - tmp_dictionary = dict() +def test_plugin_description_in_dict_multiple() -> None: + module = {"hello": {"world": {"data": PluginListPlugin, "data2": PluginListPlugin}}} - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world", "data"], PluginListPlugin)) - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world", "data2"], PluginListPlugin)) - - description = [x for x in output_plugin_description_recursive(tmp_dictionary, False)] + description = [x for x in _generate_plugin_tree_overview(module, False)] assert description == [ "hello:", " world:", @@ -60,8 +38,21 @@ def test_plugin_description_in_dict_multiple(): ] -@patch.object(plugin.plugin, "load") -@patch.object(plugin, "get_exported_plugins") -def test_categorize_plugins(mocked_export, mocked_load): - mocked_export.return_value = [{"module": "something.data"}] - assert categorize_plugins() == {"something": {"data": mocked_load.return_value}} +@patch("dissect.target.plugins.general.plugins.plugin.load") +@patch("dissect.target.plugins.general.plugins.plugin.functions") +def test_categorize_plugins(mocked_plugins: MagicMock, mocked_load: MagicMock) -> None: + mocked_plugins.return_value = [ + plugin.FunctionDescriptor( + name="data", + namespace=None, + path="something.data", + exported=True, + internal=False, + findable=True, + output=None, + method_name="data", + module="other.root.something.data", + qualname="DataClass", + ), + ] + assert _categorize_functions() == {"something": mocked_load.return_value} diff --git a/tests/plugins/os/unix/linux/test_network.py b/tests/plugins/os/unix/linux/test_network.py index e29f0c3a1..8c1381e16 100644 --- a/tests/plugins/os/unix/linux/test_network.py +++ b/tests/plugins/os/unix/linux/test_network.py @@ -6,7 +6,7 @@ from dissect.target import Target from dissect.target.filesystem import VirtualFilesystem -from dissect.target.plugins.general.network import UnixInterfaceRecord +from dissect.target.plugins.os.default.network import UnixInterfaceRecord from dissect.target.plugins.os.unix.linux.network import ( LinuxNetworkConfigParser, LinuxNetworkPlugin, diff --git a/tests/plugins/os/unix/log/test_audit.py b/tests/plugins/os/unix/log/test_audit.py index 941d7ee46..5eb205150 100644 --- a/tests/plugins/os/unix/log/test_audit.py +++ b/tests/plugins/os/unix/log/test_audit.py @@ -3,11 +3,13 @@ from dissect.util.ts import from_unix +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.os.unix.log.audit import AuditPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_audit_plugin(target_unix, fs_unix): +def test_audit_plugin(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/os/unix/log/audit/audit.log") fs_unix.map_file("var/log/audit/audit.log", data_file) @@ -32,7 +34,7 @@ def test_audit_plugin(target_unix, fs_unix): assert result.message == 'cwd="/home/shadowman"' -def test_audit_plugin_config(target_unix, fs_unix): +def test_audit_plugin_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ log_file = /foo/bar/audit/audit.log # log_file=/tmp/disabled/audit/audit.log @@ -41,8 +43,7 @@ def test_audit_plugin_config(target_unix, fs_unix): fs_unix.map_file_fh("tmp/disabled/audit/audit.log", BytesIO(b"Foo")) fs_unix.map_file_fh("foo/bar/audit/audit.log", BytesIO(b"Foo")) - audit = AuditPlugin(target_unix) - log_paths = audit.get_log_paths() + log_paths = AuditPlugin(target_unix).get_log_paths() assert len(log_paths) == 2 assert str(log_paths[0]) == "/foo/bar/audit/audit.log" assert str(log_paths[1]) == "/tmp/disabled/audit/audit.log" diff --git a/tests/plugins/os/unix/log/test_messages.py b/tests/plugins/os/unix/log/test_messages.py index 33a259cfb..85d1d2988 100644 --- a/tests/plugins/os/unix/log/test_messages.py +++ b/tests/plugins/os/unix/log/test_messages.py @@ -9,7 +9,7 @@ from dissect.target import Target from dissect.target.filesystem import VirtualFilesystem from dissect.target.filesystems.tar import TarFilesystem -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.plugins.os.unix.log.messages import MessagesPlugin, MessagesRecord from tests._utils import absolute_path @@ -73,7 +73,7 @@ def test_unix_log_messages_compressed_timezone_year_rollover() -> None: fs = TarFilesystem(bio) target.filesystems.add(fs) target.fs.mount("/", fs) - target._os_plugin = default.DefaultPlugin + target._os_plugin = DefaultPlugin target.apply() target.add_plugin(MessagesPlugin) diff --git a/tests/plugins/os/windows/test_jumplist.py b/tests/plugins/os/windows/test_jumplist.py index 354abc024..89dc08d60 100644 --- a/tests/plugins/os/windows/test_jumplist.py +++ b/tests/plugins/os/windows/test_jumplist.py @@ -34,11 +34,9 @@ def test_os_windows_jumplist(target_win_users: Target, fs_win: VirtualFilesystem target_win_users.add_plugin(JumpListPlugin) records = list(target_win_users.jumplist()) - - record = records[0] - assert len(records) == 3 + record = records[0] assert record.application_id == "590aee7bdd69b59b" assert record.application_name == "Powershell Windows 10" assert record.type == "customDestinations" diff --git a/tests/plugins/os/windows/test_mru.py b/tests/plugins/os/windows/test_mru.py index 80c65a2c2..b2df63e47 100644 --- a/tests/plugins/os/windows/test_mru.py +++ b/tests/plugins/os/windows/test_mru.py @@ -141,4 +141,4 @@ def test_mru_plugin(target_win_mru): assert len(mstsc) == 3 assert len(msoffice) == 6 - assert len(list(target_win_mru.mru.get_all_records())) == 23 + assert len(list(target_win_mru.mru())) == 23 diff --git a/tests/plugins/os/windows/test_ual.py b/tests/plugins/os/windows/test_ual.py index 983cb94a6..f45c13761 100644 --- a/tests/plugins/os/windows/test_ual.py +++ b/tests/plugins/os/windows/test_ual.py @@ -24,5 +24,5 @@ def test_ual_plugin(target_win, fs_win): domains_seen_records = list(target_win.ual.domains_seen()) assert len(domains_seen_records) == 12 - ual_all_records = list(target_win.ual.get_all_records()) + ual_all_records = list(target_win.ual()) assert len(ual_all_records) == 123 diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 10d3a3a51..9a790e961 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2,48 +2,57 @@ from functools import reduce from pathlib import Path from typing import Iterator, Optional -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest from docutils.core import publish_string from docutils.utils import SystemMessage from flow.record import Record -from dissect.target.exceptions import UnsupportedPluginError +from dissect.target.exceptions import PluginError, UnsupportedPluginError from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension from dissect.target.helpers.record import EmptyRecord, create_extended_descriptor from dissect.target.plugin import ( - PLUGINS, + FunctionDescriptor, + FunctionDescriptorLookup, InternalNamespacePlugin, InternalPlugin, NamespacePlugin, OSPlugin, Plugin, - PluginFunction, + PluginDescriptor, + PluginDescriptorLookup, + PluginRegistry, + _generate_long_paths, + _save_plugin_import_failure, alias, environment_variable_paths, export, - find_plugin_functions, + find_functions, get_external_module_paths, + load, plugins, - save_plugin_import_failure, ) -from dissect.target.plugins.general.default import DefaultPlugin +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.target import Target -def test_save_plugin_import_failure(): +@pytest.fixture(autouse=True) +def clear_caches() -> Iterator[None]: + _generate_long_paths.cache_clear() + + +def test_save_plugin_import_failure() -> None: test_trace = ["test-trace"] test_module_name = "test-module" with patch("traceback.format_exception", Mock(return_value=test_trace)): - with patch("dissect.target.plugin.PLUGINS", new_callable=dict) as MOCK_PLUGINS: - MOCK_PLUGINS["_failed"] = [] - save_plugin_import_failure(test_module_name) + with patch("dissect.target.plugin.PLUGINS", new_callable=PluginRegistry) as mock_plugins: + _save_plugin_import_failure(test_module_name) - assert len(MOCK_PLUGINS["_failed"]) == 1 - assert MOCK_PLUGINS["_failed"][0].get("module") == test_module_name - assert MOCK_PLUGINS["_failed"][0].get("stacktrace") == test_trace + assert len(mock_plugins.__failed__) == 1 + assert mock_plugins.__failed__[0].module == test_module_name + assert mock_plugins.__failed__[0].stacktrace == test_trace @pytest.mark.parametrize( @@ -54,92 +63,144 @@ def test_save_plugin_import_failure(): (":", [Path(""), Path("")]), ], ) -def test_load_environment_variable(env_value, expected_output): +def test_load_environment_variable(env_value: Optional[str], expected_output: list[Path]) -> None: with patch.object(os, "environ", {"DISSECT_PLUGINS": env_value}): assert environment_variable_paths() == expected_output -def test_load_module_paths(): +def test_load_module_paths() -> None: assert get_external_module_paths([Path(""), Path("")]) == [Path("")] -def test_load_paths_with_env(): +def test_load_paths_with_env() -> None: with patch.object(os, "environ", {"DISSECT_PLUGINS": ":"}): assert get_external_module_paths([Path(""), Path("")]) == [Path("")] class MockOSWarpPlugin(OSPlugin): __exports__ = ["f6"] # OS exports f6 - __findable__ = True + __register__ = False __name__ = "warp" def __init__(self): pass - def get_all_records(): - return [] - - def f3(self): + def f3(self) -> str: return "F3" - def f6(self): + def f6(self) -> str: return "F6" @patch( - "dissect.target.plugin.plugins", - return_value=[ - {"module": "test.x13", "exports": ["f3"], "namespace": "Warp", "class": "x13", "is_osplugin": False}, - {"module": "os", "exports": ["f3"], "namespace": None, "class": "f3", "is_osplugin": False}, - {"module": "os.warp._os", "exports": ["f6"], "namespace": None, "class": "warp", "is_osplugin": True}, - ], + "dissect.target.plugin._get_plugins", + return_value=PluginRegistry( + __functions__=FunctionDescriptorLookup( + __regular__={ + "Warp.f3": { + "test.x13.x13": FunctionDescriptor( + name="Warp.f3", + namespace="Warp", + path="test.x13.f3", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f3", + module="test.x13", + qualname="x13", + ) + }, + "f3": { + "os.f3": FunctionDescriptor( + name="f3", + namespace=None, + path="os.f3", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f3", + module="os", + qualname="f3", + ) + }, + "f22": { + "test.x69.x69": FunctionDescriptor( + name="f22", + namespace=None, + path="test.x69.f22", + exported=True, + internal=False, + findable=False, + output="record", + method_name="f22", + module="test.x69", + qualname="x69", + ) + }, + }, + __os__={ + "f6": { + "os.warp._os.warp": FunctionDescriptor( + name="f6", + namespace=None, + path="os.warp._os.f6", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f6", + module="os.warp._os", + qualname="warp", + ) + } + }, + ), + __ostree__={"os": {"warp": {}}}, + ), ) @patch("dissect.target.Target", create=True) -@patch("dissect.target.plugin.load") @pytest.mark.parametrize( - "search, findable, assert_num_found", + "search, assert_num_found", [ - ("*", True, 3), # Found with tree search using wildcard - ("*", False, 0), # Unfindable plugins are not found... - ("test.x13.*", True, 1), # Found with tree search using wildcard, expands to test.x13.f3 - ("test.x13.*", False, 0), # Unfindable plugins are not found... - ("test.x13", True, 1), # Found with tree search, same as above, because users expect +* - ("test.*", True, 1), # Found with tree search - ("test.[!x]*", True, 0), # Not Found with tree search, all in test not starting with x (no x13) - ("test.[!y]*", True, 1), # Found with tree search, all in test not starting with y (so x13 is ok) - ("test.???.??", True, 1), # Found with tree search, using question marks - ("x13", True, 0), # Not Found: Part of namespace but no match - ("Warp.*", True, 0), # Not Found: Namespace != Module so 0 - ("os.warp._os.f6", True, 1), # Found, OS-plugins also available under verbose name - ("f6", True, 1), # Found with classic search - ("f6", False, 1), # Backward compatible: unfindable has no effect on classic search - ("Warp.f3", True, 1), # Found with classic style search using namespace + function - ("Warp.f3", False, 1), # Backward compatible: unfindable has no effect on classic search - ("f3", True, 1), # Found with classic style search using only function - ("os.*", True, 2), # Found matching os.f3, os.warp._os.f6 - ("os", True, 0), # Exception for os, because it can be a 'special' plugin (tree match ignored) + ("*", 2), # Found with tree search using wildcard, excluding OS plugins and unfindable + ("test.x13.*", 1), # Found with tree search using wildcard, expands to test.x13.f3 + ("test.x13", 1), # Found with tree search, same as above, because users expect +* + ("test.x13.f3", 1), + ("test.*", 1), # Found with tree search + ("test.[!x]*", 0), # Not Found with tree search, all in test not starting with x (no x13) + ("test.[!y]*", 1), # Found with tree search, all in test not starting with y (so x13 is ok) + ("test.???.??", 1), # Found with tree search, using question marks + ("x13", 0), # Not Found: Part of namespace but no match + ("Warp.*", 0), # Not Found: Namespace != Module so 0 + ("os.warp._os.f6", 0), # OS plugins are excluded from tree search + ("f6", 1), # Found with direct match + ("f22", 1), # Unfindable has no effect on direct match + ("Warp.f3", 1), # Found with namespace + function + ("f3", 1), # Found direct match + ("os.*", 1), # Found matching os.f3 + ("os", 1), # No tree search for "os" because it's a direct match ], ) -def test_find_plugin_functions(plugin_loader, target, plugins, search, findable, assert_num_found): - os_plugin = MockOSWarpPlugin - os_plugin.__findable__ = findable - target._os_plugin = os_plugin - plugin_loader.return_value = os_plugin() +def test_find_functions(target: MagicMock, plugins: dict, search: str, assert_num_found: int) -> None: + target._os_plugin = MockOSWarpPlugin + target._os_plugin.__module__ = "dissect.target.plugins.os.warp._os" - found, _ = find_plugin_functions(target, search) + found, _ = find_functions(search, target) assert len(found) == assert_num_found -def test_find_plugin_function_windows(target_win: Target) -> None: - found, _ = find_plugin_functions(target_win, "services") +def test_find_functions_windows(target_win: Target) -> None: + found, _ = find_functions("services", target_win) assert len(found) == 1 assert found[0].name == "services" assert found[0].path == "os.windows.services.services" -def test_find_plugin_function_linux(target_linux: Target) -> None: - found, _ = find_plugin_functions(target_linux, "services") +def test_find_functions_linux(target_linux: Target) -> None: + found, _ = find_functions("services", target_linux) assert len(found) == 1 assert found[0].name == "services" @@ -153,9 +214,17 @@ def test_find_plugin_function_linux(target_linux: Target) -> None: ], ) +TestRecord2 = create_extended_descriptor([UserRecordDescriptorExtension])( + "application/test_other", + [ + ("varint", "value"), + ], +) + class _TestNSPlugin(NamespacePlugin): __namespace__ = "NS" + __register__ = False @export(record=TestRecord) def test_all(self): @@ -165,6 +234,7 @@ def test_all(self): class _TestSubPlugin1(_TestNSPlugin): __namespace__ = "t1" + __register__ = False @export(record=TestRecord) def test(self): @@ -173,6 +243,7 @@ def test(self): class _TestSubPlugin2(_TestNSPlugin): __namespace__ = "t2" + __register__ = False @export(record=TestRecord) def test(self): @@ -181,6 +252,7 @@ def test(self): class _TestSubPlugin3(_TestSubPlugin2): __namespace__ = "t3" + __register__ = False # Override the test() function of t2 @export(record=TestRecord) @@ -193,28 +265,54 @@ def _value(self): class _TestSubPlugin4(_TestSubPlugin3): __namespace__ = "t4" + __register__ = False # Do not override the test() function of t3, but change the _value function instead. def _value(self): return "test4" + @export(record=TestRecord) + def test_other(self): + yield TestRecord(value="test4-other") + @export(record=TestRecord) def test_all(self): yield TestRecord(value="overridden") +class _TestSubPlugin5(_TestNSPlugin): + __namespace__ = "t5" + __register__ = False + + @export(record=TestRecord2) + def test_other(self): + yield TestRecord2(value=69) + + def test_namespace_plugin(target_win: Target) -> None: - assert "SUBPLUGINS" in dir(_TestNSPlugin) + assert "__subplugins__" in dir(_TestNSPlugin) # Rename the test functions to protect them from being filtered by NS target_win._register_plugin_functions(_TestSubPlugin1(target_win)) target_win._register_plugin_functions(_TestSubPlugin2(target_win)) target_win._register_plugin_functions(_TestSubPlugin3(target_win)) target_win._register_plugin_functions(_TestSubPlugin4(target_win)) + target_win._register_plugin_functions(_TestSubPlugin5(target_win)) target_win._register_plugin_functions(_TestNSPlugin(target_win)) - assert len(list(target_win.NS.test())) == 4 - assert len(target_win.NS.SUBPLUGINS) == 4 + assert len(target_win.NS.__subplugins__) == 5 + assert len(target_win.NS.test.__subplugins__) == 4 + assert target_win.NS.test.__doc__ == "Return test for: t1, t2, t3, t4" + assert [rd.name for rd in target_win.NS.test.__record__] == ["application/test"] + + assert target_win.NS.test_other.__doc__ == "Return test_other for: t4, t5" + assert sorted([rd.name for rd in target_win.NS.test_other.__record__]) == [ + "application/test", + "application/test_other", + ] + assert len(target_win.NS.test_other.__subplugins__) == 2 + + assert len(list(target_win.NS.test())) == 4 assert sorted([item.value for item in target_win.NS.test()]) == ["test1", "test2", "test3", "test4"] assert sorted([item.value for item in target_win.t1.test()]) == ["test1"] assert sorted([item.value for item in target_win.t2.test()]) == ["test2"] @@ -227,12 +325,19 @@ def test_namespace_plugin(target_win: Target) -> None: # Check whether we can access the overridden function when explicitly accessing the subplugin assert next(target_win.t4.test_all()).value == "overridden" - # Remove test plugin from list afterwards to avoid order effects - del PLUGINS["tests"] + with pytest.raises(PluginError, match="Cannot merge namespace methods with different output types"): + + class _TestSubPluginFaulty(_TestNSPlugin): + __namespace__ = "faulty" + __register__ = False + + @export(output="yield") + def test(self): + yield "faulty" def test_find_plugin_function_default(target_default: Target) -> None: - found, _ = find_plugin_functions(target_default, "services") + found, _ = find_functions("services", target_default) assert len(found) == 2 names = [item.name for item in found] @@ -242,9 +347,14 @@ def test_find_plugin_function_default(target_default: Target) -> None: assert "os.unix.linux.services.services" in paths assert "os.windows.services.services" in paths - found, _ = find_plugin_functions(target_default, "mcafee.msc") + found, _ = find_functions("mcafee.msc", target_default) + assert len(found) == 1 assert found[0].path == "apps.av.mcafee.msc" + found, _ = find_functions("webserver.access", target_default) + assert len(found) == 1 + assert found[0].path == "apps.webserver.webserver.access" + @pytest.mark.parametrize( "pattern", @@ -257,7 +367,7 @@ def test_find_plugin_function_default(target_default: Target) -> None: ], ) def test_find_plugin_function_order(target_win: Target, pattern: str) -> None: - found = ",".join(reduce(lambda rs, el: rs + [el.method_name], find_plugin_functions(target_win, pattern)[0], [])) + found = ",".join(reduce(lambda rs, el: rs + [el.method_name], find_functions(pattern, target_win)[0], [])) assert found == pattern @@ -271,142 +381,301 @@ def test_incompatible_plugin(target_bare: Target) -> None: target_bare.add_plugin(_TestIncompatiblePlugin) -MOCK_PLUGINS = { - "apps": { # Plugin descriptors in this branch should be returned for any osfilter - "mail": {"module": "apps.mail", "functions": "mail"}, - }, - "os": { - # The OSPlugin for Generic OS, plugins in this branch should only be - # returned when the osfilter starts with "os." or is None. - # The _os plugin itself should only be returned if special_keys - # contains the "_os" key. - "_os": {"module": "os._os", "functions": "GenericOS"}, - "apps": { - "app1": {"module": "os.apps.app1", "functions": "app1"}, - "app2": {"module": "os.apps.app2", "functions": "app2"}, +MOCK_PLUGINS = PluginRegistry( + __functions__=FunctionDescriptorLookup( + __regular__={ + "mail": { + "apps.mail.MailPlugin": FunctionDescriptor( + name="mail", + namespace=None, + path="apps.mail.mail", + exported=True, + internal=False, + findable=True, + output="record", + method_name="mail", + module="apps.mail", + qualname="MailPlugin", + ) + }, + "app1": { + "os.apps.app1.App1Plugin": FunctionDescriptor( + name="app1", + namespace=None, + path="os.apps.app1.app1", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app1", + module="os.apps.app1", + qualname="App1Plugin", + ) + }, + "app2": { + "os.apps.app2.App2Plugin": FunctionDescriptor( + name="app2", + namespace=None, + path="os.apps.app2.app2", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app2", + module="os.apps.app2", + qualname="App2Plugin", + ), + "os.fooos.apps.app2.App2Plugin": FunctionDescriptor( + name="app2", + namespace=None, + path="os.fooos.apps.app2.app2", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app2", + module="os.fooos.apps.app2", + qualname="App2Plugin", + ), + }, + "foo_app": { + "os.fooos.apps.foo_app.FooAppPlugin": FunctionDescriptor( + name="foo_app", + namespace=None, + path="os.fooos.apps.foo_app.foo_app", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foo_app", + module="os.foos.apps.foo_app", + qualname="FooAppPlugin", + ) + }, + "bar_app": { + "os.fooos.apps.bar_app.BarAppPlugin": FunctionDescriptor( + name="bar_app", + namespace=None, + path="os.fooos.apps.bar_app.bar_app", + exported=True, + internal=False, + findable=True, + output="record", + method_name="bar_app", + module="os.foos.apps.bar_app", + qualname="BarAppPlugin", + ) + }, + "foobar": { + "os.fooos.foobar.FooBarPlugin": FunctionDescriptor( + name="foobar", + namespace=None, + path="os.fooos.foobar.foobar", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foobar", + module="os.foos.foobar", + qualname="FooBarPlugin", + ) + }, }, - "fooos": { - # The OSPlugin for FooOS, plugins in this branch should only be - # returned when the osfilter is "os.fooos" or "os.fooos._os" or - # None. - "_os": {"module": "os.foos._os", "functions": "FooOS"}, - "foobar": {"module": "os.foos.foobar", "functions": "foobar"}, - # The plugins under _misc should only be returned if special_keys - # contains the "_misc" key. - "_misc": { - "bar": {"module": "os.foos._misc.bar", "functions": "bar"}, - "tender": {"module": "os.foos._misc.tender", "functions": "tender"}, + __os__={ + "generic_os": { + "os._os.GenericOS": FunctionDescriptor( + name="generic_os", + namespace=None, + path="os._os.generic_os", + exported=True, + internal=False, + findable=True, + output="record", + method_name="generic_os", + module="os._os", + qualname="GenericOS", + ) }, - "apps": { - "foo_app": {"module": "os.foos.apps.foo_app", "functions": "foo_app"}, - "bar_app": {"module": "os.foos.apps.bar_app", "functions": "bar_app"}, + "foo_os": { + "os.fooos._os.FooOS": FunctionDescriptor( + name="foo_os", + namespace=None, + path="os.fooos._os.foo_os", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foo_os", + module="os.fooos._os", + qualname="FooOS", + ) }, }, + ), + __plugins__=PluginDescriptorLookup( + __regular__={ + "apps.mail.MailPlugin": PluginDescriptor( + module="apps.mail", + qualname="MailPlugin", + namespace=None, + path="apps.mail", + findable=True, + functions=["mail"], + exports=["mail"], + ), + "os.apps.app1.App1Plugin": PluginDescriptor( + module="os.apps.app1", + qualname="App1Plugin", + namespace=None, + path="os.apps.app1", + findable=True, + functions=["app1"], + exports=["app1"], + ), + "os.apps.app2.App2Plugin": PluginDescriptor( + module="os.apps.app2", + qualname="App2Plugin", + namespace=None, + path="os.apps.app2", + findable=True, + functions=["app2"], + exports=["app2"], + ), + "os.fooos.apps.app2.App2Plugin": PluginDescriptor( + module="os.fooos.apps.app2", + qualname="App2Plugin", + namespace=None, + path="os.fooos.apps.app2", + findable=True, + functions=["app2"], + exports=["app2"], + ), + "os.fooos.apps.foo_app.FooAppPlugin": PluginDescriptor( + module="os.fooos.apps.foo_app", + qualname="FooAppPlugin", + namespace=None, + path="os.fooos.apps.foo_app", + findable=True, + functions=["foo_app"], + exports=["foo_app"], + ), + "os.fooos.apps.bar_app.BarAppPlugin": PluginDescriptor( + module="os.fooos.apps.bar_app", + qualname="BarAppPlugin", + namespace=None, + path="os.fooos.apps.bar_app", + findable=True, + functions=["bar_app"], + exports=["bar_app"], + ), + "os.fooos.foobar.FooBarPlugin": PluginDescriptor( + module="os.fooos.foobar", + qualname="FooBarPlugin", + namespace=None, + path="os.fooos.foobar", + findable=True, + functions=["foobar"], + exports=["foobar"], + ), + }, + __os__={ + "os._os.GenericOS": PluginDescriptor( + module="os._os", + qualname="GenericOS", + namespace=None, + path="os._os", + findable=True, + functions=["generic_os"], + exports=["generic_os"], + ), + "os.fooos._os.FooOS": PluginDescriptor( + module="os.fooos._os", + qualname="FooOS", + namespace=None, + path="os.fooos._os", + findable=True, + functions=["foo_os"], + exports=["foo_os"], + ), + }, + ), + __ostree__={ + "os": { + "fooos": {}, + } }, -} +) @pytest.mark.parametrize( - "osfilter, special_keys, only_special_keys, expected_plugin_functions", + "osfilter, index, expected_plugins", [ ( None, - set(["_os", "_misc"]), - False, - [ - "mail", - "GenericOS", - "app1", - "app2", - "FooOS", - "foobar", - "bar", - "tender", - "foo_app", - "bar_app", - ], - ), - ( - "os._os", - set(["_os"]), - False, + "__regular__", [ - "mail", - "GenericOS", - "app1", - "app2", + "apps.mail", + "os.apps.app1", + "os.apps.app2", + "os.fooos.apps.app2", + "os.fooos.apps.bar_app", + "os.fooos.apps.foo_app", + "os.fooos.foobar", ], ), ( - "os.fooos._os", - set(), - False, + None, + "__os__", [ - "mail", - "app1", - "app2", - "foobar", - "foo_app", - "bar_app", + "os._os", + "os.fooos._os", ], ), ( - "os.fooos", - set(["_os"]), - False, + "os._os", + "__regular__", [ - "mail", - "app1", - "app2", - "FooOS", - "foobar", - "foo_app", - "bar_app", + "apps.mail", + "os.apps.app1", + "os.apps.app2", ], ), ( "os.fooos._os", - set(["_os", "_misc"]), - True, + "__regular__", [ - "FooOS", - "bar", - "tender", + "apps.mail", + "os.apps.app1", + "os.apps.app2", + "os.fooos.apps.app2", + "os.fooos.apps.bar_app", + "os.fooos.apps.foo_app", + "os.fooos.foobar", ], ), ( "bar", - set(["_os"]), - False, - [ - "mail", - ], + "__regular__", + ["apps.mail"], ), ], ) def test_plugins( osfilter: str, - special_keys: set[str], - only_special_keys: bool, - expected_plugin_functions: list[str], + index: str, + expected_plugins: list[str], ) -> None: with ( - patch("dissect.target.plugin.PLUGINS", MOCK_PLUGINS), - patch("dissect.target.plugin._modulepath", return_value=osfilter), + patch("dissect.target.plugin._get_plugins", return_value=MOCK_PLUGINS), + patch("dissect.target.plugin._module_path", return_value=osfilter), ): if osfilter is not None: # osfilter must be a class or None osfilter = Mock - plugin_descriptors = plugins( - osfilter=osfilter, - special_keys=special_keys, - only_special_keys=only_special_keys, - ) - - plugin_functions = [descriptor["functions"] for descriptor in plugin_descriptors] + plugin_descriptors = plugins(osfilter=osfilter, index=index) - assert sorted(plugin_functions) == sorted(expected_plugin_functions) + assert sorted([desc.module for desc in plugin_descriptors]) == sorted(expected_plugins) def test_plugins_default_plugin(target_default: Target) -> None: @@ -419,18 +688,14 @@ def test_plugins_default_plugin(target_default: Target) -> None: # target with DefaultPlugin as OS plugin. sentinel_function = "all_with_home" has_sentinel_function = False - for plugin in default_plugin_plugins: - if sentinel_function in plugin.get("functions", []): + for p in default_plugin_plugins: + if sentinel_function in p.functions: has_sentinel_function = True break assert has_sentinel_function - default_os_plugin_desc = plugins( - osfilter=target_default._os_plugin, - special_keys=set(["_os"]), - only_special_keys=True, - ) + default_os_plugin_desc = plugins(osfilter=target_default._os_plugin, index="__os__") assert len(list(default_os_plugin_desc)) == 1 @@ -452,6 +717,8 @@ def test_os_plugin_property_methods(target_bare: Target, method_name: str) -> No class MockOS1(OSPlugin): + __register__ = False + @export(property=True) def hostname(self) -> Optional[str]: pass @@ -478,6 +745,8 @@ def architecture(self) -> Optional[str]: class MockOS2(OSPlugin): + __register__ = False + @export(property=True) def hostname(self) -> Optional[str]: """Test docstring hostname""" @@ -560,7 +829,7 @@ def test(self) -> None: def test_internal_namespace_plugin() -> None: - assert "SUBPLUGINS" in dir(_TestInternalNamespacePlugin) + assert "__subplugins__" in dir(_TestInternalNamespacePlugin) assert "test" not in _TestInternalNamespacePlugin.__exports__ assert "test" in _TestInternalNamespacePlugin.__functions__ @@ -568,6 +837,8 @@ def test_internal_namespace_plugin() -> None: class ExampleFooPlugin(Plugin): """Example Foo Plugin.""" + __register__ = False + def check_compatible(self) -> None: return @@ -590,24 +861,25 @@ def test_plugin_alias(target_bare: Target) -> None: @pytest.mark.parametrize( - "func_path, func", - [(func.path, func) for func in find_plugin_functions(Target(), "*", compatibility=False, show_hidden=True)[0]], + "descriptor", + find_functions("*", Target(), compatibility=False, show_hidden=True)[0], ) -def test_exported_plugin_format(func_path: str, func: PluginFunction) -> None: +def test_exported_plugin_format(descriptor: FunctionDescriptor) -> None: """This test checks plugin style guide conformity for all exported plugins. Resources: - https://docs.dissect.tools/en/latest/contributing/style-guide.html """ + plugincls = load(descriptor) # Ignore DefaultPlugin and NamespacePlugin instances - if func.class_object.__base__ is NamespacePlugin or func.class_object is DefaultPlugin: + if plugincls.__base__ is NamespacePlugin or plugincls is DefaultPlugin: return # Plugin method should specify what it returns - assert func.output_type in ["record", "yield", "default", "none"], f"Invalid output_type for function {func}" + assert descriptor.output in ["record", "yield", "default", "none"], f"Invalid output_type for function {descriptor}" - py_func = getattr(func.class_object, func.method_name) + py_func = getattr(plugincls, descriptor.method_name) annotations = None if hasattr(py_func, "__annotations__"): @@ -617,22 +889,22 @@ def test_exported_plugin_format(func_path: str, func: PluginFunction) -> None: annotations = py_func.fget.__annotations__ # Plugin method should have a return annotation - assert annotations and "return" in annotations.keys(), f"No return type annotation for function {func}" + assert annotations and "return" in annotations.keys(), f"No return type annotation for function {descriptor}" # TODO: Check if the annotations make sense with the provided output_type # Plugin method should have a docstring method_doc_str = py_func.__doc__ - assert isinstance(method_doc_str, str), f"No docstring for function {func}" - assert method_doc_str != "", f"Empty docstring for function {func}" + assert isinstance(method_doc_str, str), f"No docstring for function {descriptor}" + assert method_doc_str != "", f"Empty docstring for function {descriptor}" # The method docstring should compile to rst without warnings assert_valid_rst(method_doc_str) # Plugin class should have a docstring - class_doc_str = func.class_object.__doc__ - assert isinstance(class_doc_str, str), f"No docstring for class {func.class_object.__name__}" - assert class_doc_str != "", f"Empty docstring for class {func.class_object.__name__}" + class_doc_str = plugincls.__doc__ + assert isinstance(class_doc_str, str), f"No docstring for class {plugincls.__name__}" + assert class_doc_str != "", f"Empty docstring for class {plugincls.__name__}" # The class docstring should compile to rst without warnings assert_valid_rst(class_doc_str) diff --git a/tests/test_registration.py b/tests/test_registration.py index d47bb4e0f..a64ad71c5 100644 --- a/tests/test_registration.py +++ b/tests/test_registration.py @@ -5,7 +5,7 @@ import pytest -from dissect.target.plugin import PLUGINS, find_py_files, load_modules_from_paths +from dissect.target.plugin import _find_py_files, load_modules_from_paths @pytest.fixture @@ -26,13 +26,13 @@ def copy_different_plugin_files(path: Path, file_name: str) -> None: def test_load_environment_variable_empty_string() -> None: - with patch("dissect.target.plugin.find_py_files") as mocked_find_py_files: + with patch("dissect.target.plugin._find_py_files") as mocked_find_py_files: load_modules_from_paths([]) mocked_find_py_files.assert_not_called() def test_load_environment_variable_comma_seperated_string() -> None: - with patch("dissect.target.plugin.find_py_files") as mocked_find_py_files: + with patch("dissect.target.plugin._find_py_files") as mocked_find_py_files: load_modules_from_paths([Path(""), Path("")]) mocked_find_py_files.assert_has_calls(calls=[call(Path(""))]) @@ -41,14 +41,14 @@ def test_filter_file(tmp_path: Path) -> None: file = tmp_path / "hello.py" file.touch() - assert list(find_py_files(file)) == [file] + assert list(_find_py_files(file)) == [file] test_file = tmp_path / "non_existent_file" - assert list(find_py_files(test_file)) == [] + assert list(_find_py_files(test_file)) == [] test_file = tmp_path / "__init__.py" test_file.touch() - assert list(find_py_files(test_file)) == [] + assert list(_find_py_files(test_file)) == [] @pytest.mark.parametrize( @@ -65,21 +65,25 @@ def test_filter_directory(tmp_path: Path, filename: str, empty_list: bool) -> No file.touch() if empty_list: - assert list(find_py_files(tmp_path)) == [] + assert list(_find_py_files(tmp_path)) == [] else: - assert file in list(find_py_files(tmp_path)) + assert file in list(_find_py_files(tmp_path)) def test_new_plugin_registration(environment_path: Path) -> None: copy_different_plugin_files(environment_path, "plugin.py") - load_modules_from_paths([environment_path]) - assert "plugin" in PLUGINS + with patch("dissect.target.plugin.register") as mock_register: + load_modules_from_paths([environment_path]) + + mock_register.assert_called_once() + assert mock_register.call_args[0][0].__name__ == "TestPlugin" def test_loader_registration(environment_path: Path) -> None: - with patch("dissect.target.loader.LOADERS", []) as mocked_loaders, patch( - "dissect.target.loader.LOADERS_BY_SCHEME", {} + with ( + patch("dissect.target.loader.LOADERS", []) as mocked_loaders, + patch("dissect.target.loader.LOADERS_BY_SCHEME", {}), ): copy_different_plugin_files(environment_path, "loader.py") load_modules_from_paths([environment_path]) diff --git a/tests/test_report.py b/tests/test_report.py index 831d75889..2f2810878 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -4,6 +4,7 @@ import pytest from dissect.target import Target +from dissect.target.plugin import FailureDescriptor, FunctionDescriptor, PluginRegistry from dissect.target.report import ( ExecutionReport, TargetExecutionReport, @@ -15,17 +16,17 @@ @pytest.fixture -def test_target(): +def test_target() -> Target: return Target("test_target") @pytest.fixture -def func_execs(): +def func_execs() -> set[str]: return {"exec2", "exec1"} @pytest.fixture -def target_execution_report(test_target, func_execs): +def target_execution_report(test_target: Target, func_execs: set[str]) -> TargetExecutionReport: return TargetExecutionReport( target=test_target, func_execs=func_execs, @@ -33,29 +34,29 @@ def target_execution_report(test_target, func_execs): @pytest.fixture -def incompatible_plugins(): +def incompatible_plugins() -> set[str]: return {"incomp_plugin2", "incomp_plugin1"} @pytest.fixture -def add_incompatible_plugins(target_execution_report, incompatible_plugins): +def add_incompatible_plugins(target_execution_report: TargetExecutionReport, incompatible_plugins: set[str]) -> None: for plugin in incompatible_plugins: target_execution_report.add_incompatible_plugin(plugin) @pytest.fixture -def registered_plugins(): +def registered_plugins() -> set[str]: return {"regist_plugin2", "regist_plugin1"} @pytest.fixture -def add_registered_plugins(target_execution_report, registered_plugins): +def add_registered_plugins(target_execution_report: TargetExecutionReport, registered_plugins: set[str]) -> None: for plugin in registered_plugins: target_execution_report.add_registered_plugin(plugin) @pytest.fixture -def func_errors(): +def func_errors() -> dict[str, str]: return { "func1": "trace1", "func2": "trace2", @@ -63,49 +64,49 @@ def func_errors(): @pytest.fixture -def add_func_errors(target_execution_report, func_errors): +def add_func_errors(target_execution_report: TargetExecutionReport, func_errors: dict[str, str]) -> None: for func, trace in func_errors.items(): target_execution_report.add_func_error(func, trace) def test_target_execution_report_add_incompatible_plugin( - target_execution_report, - add_incompatible_plugins, - incompatible_plugins, -): + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + incompatible_plugins: set[str], +) -> None: for plugin in target_execution_report.incompatible_plugins: assert plugin in incompatible_plugins def test_target_execution_report_add_registered_plugin( - target_execution_report, - add_registered_plugins, - registered_plugins, -): + target_execution_report: TargetExecutionReport, + add_registered_plugins: None, + registered_plugins: set[str], +) -> None: for plugin in target_execution_report.registered_plugins: assert plugin in registered_plugins def test_target_execution_report_add_func_error( - target_execution_report, - add_func_errors, - func_errors, -): + target_execution_report: TargetExecutionReport, + add_func_errors: None, + func_errors: dict[str, str], +) -> None: for func, trace in target_execution_report.func_errors.items(): assert func_errors.get(func) == trace def test_target_execution_report_as_dict( - test_target, - target_execution_report, - add_incompatible_plugins, - add_registered_plugins, - add_func_errors, - incompatible_plugins, - registered_plugins, - func_errors, - func_execs, -): + test_target: Target, + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + add_registered_plugins: None, + add_func_errors: None, + incompatible_plugins: set[str], + registered_plugins: set[str], + func_errors: dict[str, str], + func_execs: set[str], +) -> None: report_dict = target_execution_report.as_dict() assert report_dict.get("target") == str(test_target) assert report_dict.get("incompatible_plugins") == sorted(incompatible_plugins) @@ -115,63 +116,63 @@ def test_target_execution_report_as_dict( @pytest.fixture -def execution_report(): +def execution_report() -> ExecutionReport: return ExecutionReport() @pytest.fixture -def cli_args(): +def cli_args() -> argparse.Namespace: return argparse.Namespace(foo="bar", baz="bla") @pytest.fixture -def set_cli_args(execution_report, cli_args): +def set_cli_args(execution_report: ExecutionReport, cli_args: argparse.Namespace) -> None: execution_report.set_cli_args(cli_args) @pytest.fixture -def plugin_stats(): - return { - "_failed": [ - { - "module": "plugin1", - "stacktrace": "trace1", - }, - { - "module": "plugin2", - "stacktrace": "trace2", - }, +def plugin_stats() -> PluginRegistry: + return PluginRegistry( + __failed__=[ + FailureDescriptor( + module="plugin1", + stacktrace="trace1", + ), + FailureDescriptor( + module="plugin2", + stacktrace="trace2", + ), ] - } + ) @pytest.fixture -def set_plugin_stats(execution_report, plugin_stats): +def set_plugin_stats(execution_report: ExecutionReport, plugin_stats: PluginRegistry) -> None: execution_report.set_plugin_stats(plugin_stats) @pytest.fixture -def target1(): +def target1() -> Target: return Target("test1") @pytest.fixture -def target2(): +def target2() -> Target: return Target("test2") @pytest.fixture -def target_report1(execution_report, target1): +def target_report1(execution_report: ExecutionReport, target1: Target) -> TargetExecutionReport: return execution_report.add_target_report(target1) @pytest.fixture -def target_report2(execution_report, target2): +def target_report2(execution_report: ExecutionReport, target2: Target) -> TargetExecutionReport: return execution_report.add_target_report(target2) @pytest.fixture -def plugin1(): +def plugin1() -> MagicMock: plugin1 = MagicMock() plugin1.__module__ = "test_module" plugin1.__qualname__ = "plugin1" @@ -179,32 +180,32 @@ def plugin1(): def test_execution_report_set_cli_args( - execution_report, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: assert execution_report.cli_args == vars(cli_args) def test_execution_report_set_plugin_stats( - execution_report, - set_plugin_stats, - plugin_stats, -): - failed_plugins = plugin_stats["_failed"] + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: PluginRegistry, +) -> None: + failed_plugins = plugin_stats.__failed__ assert len(execution_report.plugin_import_errors) == len(failed_plugins) for failed_plugin in failed_plugins: - module = failed_plugin["module"] - stacktrace = failed_plugin["stacktrace"] + module = failed_plugin.module + stacktrace = failed_plugin.stacktrace assert execution_report.plugin_import_errors.get(module) == stacktrace def test_execution_report_get_formatted_report( - execution_report, - target_report1, - target_report2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, +) -> None: with patch("dissect.target.report.make_cli_args_overview", return_value="line_1"): with patch("dissect.target.report.make_plugin_import_errors_overview", return_value="line_2"): with patch("dissect.target.report.format_target_report", return_value="line_x"): @@ -212,22 +213,22 @@ def test_execution_report_get_formatted_report( def test_execution_report_add_target_report( - execution_report, - target_report1, - target_report2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, +) -> None: assert len(execution_report.target_reports) == 2 assert target_report1 in execution_report.target_reports assert target_report2 in execution_report.target_reports def test_execution_report_get_target_report( - execution_report, - target_report1, - target_report2, - target1, - target2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, + target1: Target, + target2: Target, +) -> None: assert target_report1 == execution_report.get_target_report(target1) assert target_report2 == execution_report.get_target_report(target2) target3 = Target("nope") @@ -236,62 +237,73 @@ def test_execution_report_get_target_report( assert target_report3.target == target3 -def test_execution_report__get_plugin_name(execution_report, plugin1): +def test_execution_report__get_plugin_name(execution_report: ExecutionReport, plugin1: MagicMock) -> None: assert execution_report._get_plugin_name(plugin1) == "test_module.plugin1" def test_execution_report_log_incompatible_plugin_plugin_cls( - execution_report, - target_report1, - target1, - plugin1, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + plugin1: MagicMock, +) -> None: execution_report.log_incompatible_plugin(target1, None, plugin_cls=plugin1) assert "test_module.plugin1" in target_report1.incompatible_plugins def test_execution_report_log_incompatible_plugin_plugin_desc( - execution_report, - target_report1, - target1, -): - plugin_desc = {"fullname": "test_module.plugin1"} + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, +) -> None: + plugin_desc = FunctionDescriptor( + name="plugin1", + namespace=None, + path="", + exported=True, + internal=False, + findable=True, + output=None, + method_name="plugin1", + module="test_module", + qualname="plugin1", + ) execution_report.log_incompatible_plugin(target1, None, plugin_desc=plugin_desc) assert "test_module.plugin1" in target_report1.incompatible_plugins def test_execution_report_log_registered_plugin( - execution_report, - target_report1, - target1, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, +) -> None: execution_report.log_registered_plugin(target1, None, plugin_inst=MagicMock()) assert "unittest.mock.MagicMock" in target_report1.registered_plugins def test_execution_report_log_func_error( - execution_report, - target_report1, - target1, - func_errors, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + func_errors: dict[str, str], +) -> None: func, trace = next(iter(func_errors.items())) execution_report.log_func_error(target1, None, func, trace) assert target_report1.func_errors.get(func) == trace def test_execution_report_log_func_execution( - execution_report, - target_report1, - target1, - func_execs, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + func_execs: set[str], +) -> None: func = next(iter(func_execs)) execution_report.log_func_execution(target1, None, func) assert func in target_report1.func_execs -def test_execution_report_set_event_callbacks(execution_report): +def test_execution_report_set_event_callbacks(execution_report: ExecutionReport) -> None: mock_target = MagicMock() event_callbacks = ( (Event.INCOMPATIBLE_PLUGIN, execution_report.log_incompatible_plugin), @@ -309,14 +321,14 @@ def test_execution_report_set_event_callbacks(execution_report): def test_execution_report_as_dict( - execution_report, - set_plugin_stats, - plugin_stats, - target_report1, - target_report2, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: PluginRegistry, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: expected_dict = { "plugin_import_errors": { "plugin1": "trace1", @@ -336,36 +348,36 @@ def test_execution_report_as_dict( def test_report_make_cli_args_overview( - execution_report, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: cli_args_overview = make_cli_args_overview(execution_report) assert "foo: bar" in cli_args_overview assert "baz: bla" in cli_args_overview def test_report_make_plugin_import_errors_overview( - execution_report, - set_plugin_stats, - plugin_stats, -): + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: PluginRegistry, +) -> None: plugin_import_errors_overview = make_plugin_import_errors_overview(execution_report) assert "plugin1:\n trace1" in plugin_import_errors_overview assert "plugin2:\n trace2" in plugin_import_errors_overview def test_report_format_target_report( - test_target, - target_execution_report, - add_incompatible_plugins, - add_registered_plugins, - add_func_errors, - incompatible_plugins, - registered_plugins, - func_errors, - func_execs, -): + test_target: Target, + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + add_registered_plugins: None, + add_func_errors: None, + incompatible_plugins: set[str], + registered_plugins: set[str], + func_errors: dict[str, str], + func_execs: set[str], +) -> None: target_report = format_target_report(target_execution_report) assert str(test_target) in target_report diff --git a/tests/tools/conftest.py b/tests/tools/conftest.py new file mode 100644 index 000000000..fdcedf697 --- /dev/null +++ b/tests/tools/conftest.py @@ -0,0 +1,9 @@ +import pytest + +from dissect.target import loader + + +@pytest.fixture(scope="module", autouse=True) +def reset_loaders() -> None: + for ldr in loader.LOADERS: + ldr.module._module = None diff --git a/tests/tools/test_build_pluginlist.py b/tests/tools/test_build_pluginlist.py new file mode 100644 index 000000000..55cbc7112 --- /dev/null +++ b/tests/tools/test_build_pluginlist.py @@ -0,0 +1,27 @@ +import argparse +from unittest.mock import patch + +from dissect.target.plugin import PluginRegistry +from dissect.target.tools import build_pluginlist + + +def test_main_output() -> None: + with patch("argparse.ArgumentParser.parse_args", return_value=argparse.Namespace(verbose=0)): + with patch("dissect.target.tools.build_pluginlist.plugin.generate", return_value=PluginRegistry()): + with patch("builtins.print") as mock_print: + build_pluginlist.main() + + expected_output = """ +from dissect.target.plugin import ( + FailureDescriptor, + FunctionDescriptor, + FunctionDescriptorLookup, + PluginDescriptor, + PluginDescriptorLookup, + PluginRegistry, +) + +PLUGINS = PluginRegistry(__plugins__=PluginDescriptorLookup(__regular__={}, __os__={}, __child__={}), __functions__=FunctionDescriptorLookup(__regular__={}, __os__={}, __child__={}), __ostree__={}, __failed__=[]) +""" # noqa: E501 + + mock_print.assert_called_with(expected_output) diff --git a/tests/tools/test_query.py b/tests/tools/test_query.py index d8f51b68a..83320e84b 100644 --- a/tests/tools/test_query.py +++ b/tests/tools/test_query.py @@ -1,17 +1,19 @@ +from __future__ import annotations + import json import os import re -from typing import Any, Optional -from unittest.mock import MagicMock, patch +from typing import Any +from unittest.mock import patch import pytest -from dissect.target.plugin import PluginFunction +from dissect.target.plugin import FunctionDescriptor from dissect.target.target import Target from dissect.target.tools.query import main as target_query -def test_target_query_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: +def test_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m: m.setattr("sys.argv", ["target-query", "--list"]) @@ -20,7 +22,7 @@ def test_target_query_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.Mo out, _ = capsys.readouterr() assert out.startswith("Available plugins:") - assert "Failed to load:\n None\nAvailable loaders:\n" in out + assert "Failed to load:\n None\n\nAvailable loaders:\n" in out @pytest.mark.parametrize( @@ -44,11 +46,11 @@ def test_target_query_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.Mo ), ( ["apps.webserver.iis.doesnt.exist", "apps.webserver.apache.access"], - ["apps.webserver.iis.doesnt.exist*"], + ["apps.webserver.iis.doesnt.exist"], ), ], ) -def test_target_query_invalid_functions( +def test_invalid_functions( capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, given_funcs: list[str], @@ -97,11 +99,11 @@ def test_target_query_invalid_functions( ), ( ["apps.webserver.iis.doesnt.exist", "apps.webserver.apache.access"], - ["apps.webserver.iis.doesnt.exist*"], + ["apps.webserver.iis.doesnt.exist"], ), ], ) -def test_target_query_invalid_excluded_functions( +def test_invalid_excluded_functions( capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, given_funcs: list[str], @@ -136,7 +138,7 @@ def test_target_query_invalid_excluded_functions( assert invalid_funcs == expected_invalid_funcs -def test_target_query_unsupported_plugin_log(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: +def test_unsupported_plugin_log(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m: m.setattr( "sys.argv", @@ -149,22 +151,21 @@ def test_target_query_unsupported_plugin_log(capsys: pytest.CaptureFixture, monk assert "Unsupported plugin for regf: Registry plugin not loaded" in err -def mock_find_plugin_function( - target: Target, - patterns: str, - compatibility: bool = False, - **kwargs, -) -> tuple[list[PluginFunction], set[str]]: +def mock_find_functions(patterns: str, *args, **kwargs) -> tuple[list[FunctionDescriptor], set[str]]: plugins = [] for pattern in patterns.split(","): plugins.append( - PluginFunction( + FunctionDescriptor( name=pattern, - output_type="record", + namespace=None, path=pattern, - class_object=MagicMock(), + exported=True, + internal=False, + findable=True, + output="record", method_name=pattern, - plugin_desc={}, + module=pattern, + qualname=pattern.capitalize(), ), ) @@ -173,13 +174,13 @@ def mock_find_plugin_function( def mock_execute_function( target: Target, - func: PluginFunction, - cli_params: Optional[list[str]] = None, + func: FunctionDescriptor, + arguments: list[str] | None = None, ) -> tuple[str, Any, list[str]]: - return (func.output_type, func.name, "") + return (func.output, func.name, "") -def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> None: +def test_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m: m.setattr( "sys.argv", @@ -195,14 +196,14 @@ def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> Non with ( patch( - "dissect.target.tools.query.find_plugin_functions", + "dissect.target.tools.query.find_functions", autospec=True, - side_effect=mock_find_plugin_function, + side_effect=mock_find_functions, ), patch( - "dissect.target.tools.utils.find_plugin_functions", + "dissect.target.tools.utils.find_functions", autospec=True, - side_effect=mock_find_plugin_function, + side_effect=mock_find_functions, ), patch( "dissect.target.tools.query.execute_function_on_target", @@ -224,7 +225,7 @@ def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> Non assert executed_func_names == {"foo", "bar"} -def test_target_query_dry_run(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: +def test_dry_run(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: if os.sep == "\\": target_file = "tests\\_data\\loaders\\tar\\test-archive.tar.gz" else: @@ -239,12 +240,7 @@ def test_target_query_dry_run(capsys: pytest.CaptureFixture, monkeypatch: pytest target_query() out, _ = capsys.readouterr() - assert out == ( - f"Dry run on: \n" - " execute: users (general.default.users)\n" - " execute: network.interfaces (general.network.interfaces)\n" - " execute: osinfo (general.osinfo.osinfo)\n" - ) + assert out == (f"Dry run on: \n execute: osinfo (general.osinfo.osinfo)\n") def test_target_query_list_json(capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch) -> None: @@ -268,8 +264,8 @@ def test_target_query_list_json(capsys: pytest.CaptureFixture, monkeypatch: pyte assert len(output["plugins"]["loaded"]) > 200, "Expected more loaded plugins" assert not output["plugins"].get("failed"), "Some plugin(s) failed to initialize" - def get_plugin(plugins: list[dict], needle: str) -> dict: - match = [p for p in output["plugins"]["loaded"] if p["name"] == needle] + def get_plugin(plugins: list[dict], needle: str) -> dict | bool: + match = [p for p in plugins["plugins"]["loaded"] if p["name"] == needle] return match[0] if match else False # general plugin @@ -278,7 +274,8 @@ def get_plugin(plugins: list[dict], needle: str) -> dict: "name": "users", "description": "Return the users available in the target.", "output": "record", - "path": "general.default.users", + "arguments": [], + "path": "os.default._os.users", } # namespaced plugin @@ -287,6 +284,7 @@ def get_plugin(plugins: list[dict], needle: str) -> dict: "name": "plocate.locate", "description": "Yield file and directory names from the plocate.db.", "output": "record", + "arguments": [], "path": "os.unix.locate.plocate.locate", } @@ -296,5 +294,6 @@ def get_plugin(plugins: list[dict], needle: str) -> dict: "name": "sam", "description": "Dump SAM entries", "output": "record", + "arguments": [], "path": "os.windows.credential.sam.sam", } diff --git a/tests/tools/test_utils.py b/tests/tools/test_utils.py index db5975014..2ee264ada 100644 --- a/tests/tools/test_utils.py +++ b/tests/tools/test_utils.py @@ -1,20 +1,25 @@ +from __future__ import annotations + from datetime import datetime from pathlib import Path +from typing import TYPE_CHECKING from unittest.mock import patch import pytest from dissect.target.exceptions import UnsupportedPluginError -from dissect.target.plugin import arg, find_plugin_functions +from dissect.target.plugin import arg, find_functions from dissect.target.tools.utils import ( args_to_uri, generate_argparse_for_unbound_method, - get_target_attribute, persist_execution_report, ) +if TYPE_CHECKING: + from dissect.target.target import Target + -def test_persist_execution_report(): +def test_persist_execution_report() -> None: output_path = Path("/tmp/test/path") report_data = { "item1": { @@ -62,12 +67,12 @@ class FakeLoader: @pytest.mark.parametrize( "pattern, expected_function", [ - ("passwords", "dissect.target.plugins.os.unix.shadow.ShadowPlugin"), - ("firefox.passwords", "Unsupported function `firefox` for target"), + ("passwords", "dissect.target.plugins.os.unix.shadow"), + ("firefox.passwords", "Unsupported function `firefox.passwords`"), ], ) -def test_plugin_name_confusion_regression(target_unix_users, pattern, expected_function): - plugins, _ = find_plugin_functions(target_unix_users, pattern) +def test_plugin_name_confusion_regression(target_unix_users: Target, pattern: str, expected_function: str) -> None: + plugins, _ = find_functions(pattern, target_unix_users) assert len(plugins) == 1 # We don't expect these functions to work since our target_unix_users fixture @@ -75,7 +80,7 @@ def test_plugin_name_confusion_regression(target_unix_users, pattern, expected_f # only interested in the plugin or namespace that was called so we check # the exception stack trace. with pytest.raises(UnsupportedPluginError) as exc_info: - get_target_attribute(target_unix_users, plugins[0]) + target_unix_users.get_function(plugins[0]) assert expected_function in str(exc_info.value)