Restructure tests and some plugins (fox-it#426)

.gitattributes

@@ -1,11 +1,11 @@
-tests/data/enc-volume.bin filter=lfs diff=lfs merge=lfs -text
-tests/data/uppercase_driveletter.tar filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/browsers/iexplore/WebCacheV01.dat.gz filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/browsers/firefox/places.sqlite filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/browsers/chrome/History.sqlite filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/browsers/edge/History.sqlite filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/browsers/chromium/History.sqlite filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/os/windows/dpapi/** filter=lfs diff=lfs merge=lfs -text
-tests/data/volumes/md-nested.bin.gz filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/os/windows/notifications/appdb.dat.v3.gz filter=lfs diff=lfs merge=lfs -text
-tests/data/plugins/os/windows/notifications/wpndatabase.db filter=lfs diff=lfs merge=lfs -text
+tests/_data/loaders/tar/uppercase_driveletter.tar filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/apps/browser/iexplore/WebCacheV01.dat.gz filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/apps/browser/firefox/places.sqlite filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/apps/browser/chrome/History.sqlite filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/apps/browser/edge/History.sqlite filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/apps/browser/chromium/History.sqlite filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/os/windows/dpapi/** filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/os/windows/notifications/appdb.dat.v3.gz filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/os/windows/notifications/wpndatabase.db filter=lfs diff=lfs merge=lfs -text
+tests/_data/volumes/bde/enc-volume.bin filter=lfs diff=lfs merge=lfs -text
+tests/_data/volumes/md/md-nested.bin.gz filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -6,6 +6,6 @@ dist/
 *.pyc
 __pycache__/
 .pytest_cache/
-tests/docs/api
-tests/docs/build
-.tox/
+tests/_docs/api
+tests/_docs/build
+.tox/

MANIFEST.in

@@ -1,4 +1,4 @@
 exclude .gitattributes
 exclude .gitignore
 recursive-exclude .github/ *
-recursive-exclude tests/data/ *
+recursive-exclude tests/_data/ *

dissect/target/plugins/browsers/chrome.py → dissect/target/plugins/apps/browser/chrome.py

@@ -1,13 +1,13 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import export
-from dissect.target.plugins.browsers.browser import (
+from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_EXTENSION_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
     BrowserPlugin,
 )
-from dissect.target.plugins.browsers.chromium import (
+from dissect.target.plugins.apps.browser.chromium import (
     CHROMIUM_DOWNLOAD_RECORD_FIELDS,
     ChromiumMixin,
 )

dissect/target/plugins/browsers/chromium.py → dissect/target/plugins/apps/browser/chromium.py

@@ -13,7 +13,7 @@
 from dissect.target.helpers.fsutil import TargetPath
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import export
-from dissect.target.plugins.browsers.browser import (
+from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_EXTENSION_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,

dissect/target/plugins/browsers/edge.py → dissect/target/plugins/apps/browser/edge.py

@@ -1,13 +1,13 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import export
-from dissect.target.plugins.browsers.browser import (
+from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_EXTENSION_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
     BrowserPlugin,
 )
-from dissect.target.plugins.browsers.chromium import (
+from dissect.target.plugins.apps.browser.chromium import (
     CHROMIUM_DOWNLOAD_RECORD_FIELDS,
     ChromiumMixin,
 )

dissect/target/plugins/browsers/firefox.py → dissect/target/plugins/apps/browser/firefox.py

@@ -11,7 +11,7 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import export
-from dissect.target.plugins.browsers.browser import (
+from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
     BrowserPlugin,

dissect/target/plugins/browsers/iexplore.py → dissect/target/plugins/apps/browser/iexplore.py

@@ -8,7 +8,7 @@
 from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension
 from dissect.target.helpers.record import create_extended_descriptor
 from dissect.target.plugin import export
-from dissect.target.plugins.browsers.browser import (
+from dissect.target.plugins.apps.browser.browser import (
     GENERIC_DOWNLOAD_RECORD_FIELDS,
     GENERIC_HISTORY_RECORD_FIELDS,
     BrowserPlugin,

dissect/target/plugins/apps/webservers/apache.py → dissect/target/plugins/apps/webserver/apache.py

@@ -8,7 +8,7 @@
 from dissect.target import plugin
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.fsutil import open_decompress
-from dissect.target.plugins.apps.webservers.webservers import WebserverAccessLogRecord
+from dissect.target.plugins.apps.webserver.webserver import WebserverAccessLogRecord
 from dissect.target.target import Target
 
 COMMON_REGEX = r'(?P<remote_ip>.*?) (?P<remote_logname>.*?) (?P<remote_user>.*?) \[(?P<ts>.*)\] "(?P<method>.*?) (?P<uri>.*?) ?(?P<protocol>HTTP\/.*?)?" (?P<status_code>\d{3}) (?P<bytes_sent>-|\d+)'  # noqa: E501

dissect/target/plugins/apps/webservers/caddy.py → dissect/target/plugins/apps/webserver/caddy.py

@@ -9,7 +9,7 @@
 from dissect.target import plugin
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.fsutil import basename, open_decompress
-from dissect.target.plugins.apps.webservers.webservers import WebserverAccessLogRecord
+from dissect.target.plugins.apps.webserver.webserver import WebserverAccessLogRecord
 from dissect.target.target import Target
 
 LOG_FILE_REGEX = re.compile(r"(log|output file) (?P<log_file>.*)( \{)?$")

dissect/target/plugins/apps/webservers/iis.py → dissect/target/plugins/apps/webserver/iis.py

@@ -12,7 +12,7 @@
 from dissect.target.exceptions import PluginError, UnsupportedPluginError
 from dissect.target.helpers import fsutil
 from dissect.target.helpers.record import TargetRecordDescriptor
-from dissect.target.plugins.apps.webservers.webservers import WebserverAccessLogRecord
+from dissect.target.plugins.apps.webserver.webserver import WebserverAccessLogRecord
 
 LOG_RECORD_NAME = "filesystem/windows/iis/logs"
 

dissect/target/plugins/apps/webservers/nginx.py → dissect/target/plugins/apps/webserver/nginx.py

@@ -6,7 +6,7 @@
 from dissect.target import plugin
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
 from dissect.target.helpers.fsutil import open_decompress
-from dissect.target.plugins.apps.webservers.webservers import WebserverAccessLogRecord
+from dissect.target.plugins.apps.webserver.webserver import WebserverAccessLogRecord
 from dissect.target.target import Target
 
 LOG_REGEX = re.compile(

tests/conftest.py

@@ -19,12 +19,11 @@
 from dissect.target.plugins.os.windows import registry
 from dissect.target.plugins.os.windows._os import WindowsPlugin
 from dissect.target.target import Target
-
-from ._utils import absolute_path
+from tests._utils import absolute_path
 
 # Test if the data/ directory is present and if not, as is the case in Python
 # source distributions of dissect.target, we give an error
-data_dir = absolute_path("data")
+data_dir = absolute_path("_data")
 if not pathlib.Path(data_dir).is_dir():
     raise pytest.PytestConfigWarning(
         f"No test data directory {data_dir} found.\n"
@@ -137,7 +136,7 @@ def fs_linux_proc_sockets(fs_linux_proc: VirtualFilesystem) -> VirtualFilesystem
     fs = fs_linux_proc
 
     for filename in ("unix", "packet", "raw6", "raw", "udp6", "udp", "tcp6", "tcp"):
-        fs.map_file(f"/proc/net/{filename}", absolute_path(f"data/unix/linux/proc/net/{filename}"))
+        fs.map_file(f"/proc/net/{filename}", absolute_path(f"_data/plugins/os/unix/linux/proc/net/{filename}"))
 
     yield fs
 
@@ -153,14 +152,14 @@ def fs_osx():
 @pytest.fixture
 def fs_bsd():
     fs = VirtualFilesystem()
-    fs.map_file("/bin/freebsd-version", absolute_path("data/plugins/os/unix/bsd/freebsd/freebsd-freebsd-version"))
+    fs.map_file("/bin/freebsd-version", absolute_path("_data/plugins/os/unix/bsd/freebsd/freebsd-freebsd-version"))
     yield fs
 
 
 @pytest.fixture
 def fs_android() -> VirtualFilesystem:
     fs = VirtualFilesystem()
-    fs.map_file("/build.prop", absolute_path("data/plugins/os/unix/linux/android/build.prop"))
+    fs.map_file("/build.prop", absolute_path("_data/plugins/os/unix/linux/android/build.prop"))
     yield fs
 
 
@@ -250,10 +249,10 @@ def target_osx(fs_osx):
     mock_target.fs.mount("/", fs_osx)
     mock_target.apply()
 
-    version = absolute_path("data/plugins/os/unix/bsd/osx/os/SystemVersion.plist")
+    version = absolute_path("_data/plugins/os/unix/bsd/osx/_os/SystemVersion.plist")
     fs_osx.map_file("/System/Library/CoreServices/SystemVersion.plist", version)
 
-    system = absolute_path("data/plugins/os/unix/bsd/osx/os/preferences.plist")
+    system = absolute_path("_data/plugins/os/unix/bsd/osx/_os/preferences.plist")
     fs_osx.map_file("/Library/Preferences/SystemConfiguration/preferences.plist", system)
 
     yield mock_target
@@ -271,7 +270,7 @@ def target_citrix(fs_bsd):
     mock_target.filesystems.add(var_filesystem)
 
     flash_filesystem = VirtualFilesystem()
-    flash_filesystem.map_dir("/", absolute_path("data/plugins/os/unix/bsd/citrix/flash"))
+    flash_filesystem.map_dir("/", absolute_path("_data/plugins/os/unix/bsd/citrix/_os/flash"))
     mock_target.filesystems.add(flash_filesystem)
 
     mock_target.apply()
@@ -390,10 +389,10 @@ def target_linux_users(target_linux: Target, fs_linux: VirtualFilesystem) -> Tar
 
 @pytest.fixture
 def target_osx_users(target_osx, fs_osx):
-    dissect = absolute_path("data/plugins/os/unix/bsd/osx/os/dissect.plist")
+    dissect = absolute_path("_data/plugins/os/unix/bsd/osx/_os/dissect.plist")
     fs_osx.map_file("/var/db/dslocal/nodes/Default/users/_dissect.plist", dissect)
 
-    test = absolute_path("data/plugins/os/unix/bsd/osx/os/test.plist")
+    test = absolute_path("_data/plugins/os/unix/bsd/osx/_os/test.plist")
     fs_osx.map_file("/var/db/dslocal/nodes/Default/users/_test.plist", test)
 
     yield target_osx

tests/test_filesystems_config.py → tests/filesystems/test_config.py

@@ -12,15 +12,14 @@
     ConfigurationFilesystem,
 )
 from dissect.target.helpers.configutil import parse_config
-
-from ._utils import absolute_path
+from tests._utils import absolute_path
 
 
 @pytest.fixture
 def etc_directory(tmp_path: Path, fs_unix: VirtualFilesystem) -> VirtualFilesystem:
     tmp_path.joinpath("new/path").mkdir(parents=True, exist_ok=True)
     tmp_path.joinpath("new/config").mkdir(parents=True, exist_ok=True)
-    tmp_path.joinpath("new/path/config").write_text(Path(absolute_path("data/helpers/configparser/config")).read_text())
+    tmp_path.joinpath("new/path/config").write_text(Path(absolute_path("_data/helpers/configutil/config")).read_text())
     fs_unix.map_dir("/etc", tmp_path)
 
     return fs_unix
@@ -38,27 +37,27 @@ def mapped_file(test_file: str, fs_unix: VirtualFilesystem) -> VirtualFilesystem
     "test_file, expected_output",
     [
         (
-            "data/helpers/configparser/hosts",
+            "_data/helpers/configutil/hosts",
             {
                 "127.0.0.1": "localhost",
                 "::1": "localhost",
                 "127.0.1.1": "pop-os.localdomain pop-os",
             },
         ),
         (
-            "data/helpers/configparser/hosts.allow",
+            "_data/helpers/configutil/hosts.allow",
             {"ALL": ["LOCAL @some_netgroup", ".foobar.edu EXCEPT terminalserver.foobar.edu"]},
         ),
         (
-            "data/helpers/configparser/hosts.deny",
+            "_data/helpers/configutil/hosts.deny",
             {"ALL": "PARANOID", "ALL EXCEPT in.fingerd": "other.host.name, .other.domain"},
         ),
         (
-            "data/helpers/configparser/resolv.conf",
+            "_data/helpers/configutil/resolv.conf",
             {"nameserver": "127.0.0.53", "options": "edns0", "search": "local"},
         ),
         (
-            "data/helpers/configparser/sshd_config",
+            "_data/helpers/configutil/sshd_config",
             {
                 "HostKey": [f"__PROGRAMDATA__/ssh/ssh_host_{key}_key" for key in ["rsa", "dsa", "ecdsa", "ed25519"]],
                 "AuthorizedKeysFile": ".ssh/authorized_keys",

tests/test_helpers_docs.py → tests/helpers/test_docs.py

@@ -1,5 +1,5 @@
 from dissect.target.helpers import docs
-from dissect.target.plugins.apps.webservers.iis import IISLogsPlugin
+from dissect.target.plugins.apps.webserver.iis import IISLogsPlugin
 
 
 def get_nonempty_lines_set(paragraph):

tests/test_helpers_fsutil.py → tests/helpers/test_fsutil.py

@@ -3,12 +3,14 @@
 import io
 import os
 import platform
+import tempfile
 from contextlib import contextmanager
 from unittest.mock import Mock, patch
 
 import pytest
 
-from dissect.target.filesystem import VirtualFilesystem
+from dissect.target.filesystem import VirtualFile, VirtualFilesystem
+from dissect.target.filesystems.dir import DirectoryFilesystem
 from dissect.target.helpers import fsutil
 
 
@@ -369,3 +371,52 @@ def no_listxattr():
 def test_fs_attrs_no_os_listxattr():
     with no_listxattr():
         assert fsutil.fs_attrs("/some/path") == {}
+
+
+def test_target_path_checks_dirfs(tmp_path, target_win):
+    with tempfile.NamedTemporaryFile(dir=tmp_path) as tf:
+        tf.write(b"dummy")
+        tf.flush()
+        tmpfile_name = os.path.basename(tf.name)
+
+        fs = DirectoryFilesystem(path=tmp_path)
+        target_win.filesystems.add(fs)
+        target_win.fs.mount("Z:\\", fs)
+        assert target_win.fs.path(f"Z:\\{tmpfile_name}").is_file()
+        assert not target_win.fs.path(f"Z:\\{tmpfile_name}\\some").exists()
+        assert not target_win.fs.path(f"Z:\\{tmpfile_name}\\some").is_file()
+
+
+def test_target_path_checks_mapped_dir(tmp_path, target_win):
+    with tempfile.NamedTemporaryFile(dir=tmp_path) as tf:
+        tf.write(b"dummy")
+        tf.flush()
+        tmpfile_name = os.path.basename(tf.name)
+
+        target_win.filesystems.entries[0].map_dir("test-dir", tmp_path)
+        assert target_win.fs.path("C:\\test-dir\\").is_dir()
+        assert not target_win.fs.path("C:\\test-dir\\").is_file()
+
+        assert target_win.fs.path(f"C:\\test-dir\\{tmpfile_name}").is_file()
+        assert not target_win.fs.path(f"C:\\test-dir\\{tmpfile_name}\\some").is_file()
+
+
+def test_target_path_checks_virtual():
+    vfs = VirtualFilesystem()
+    vfs.map_file_entry("file", VirtualFile(vfs, "file", None))
+    assert not vfs.path("file/test").exists()
+
+
+def test_target_path_backslash_normalisation(target_win, fs_win, tmp_path):
+    with tempfile.NamedTemporaryFile(dir=tmp_path) as tf:
+        tf.write(b"dummy")
+        tf.flush()
+
+        fs_win.map_dir("windows/system32/", tmp_path)
+        fs_win.map_file("windows/system32/somefile.txt", tf.name)
+
+        results = list(target_win.fs.path("/").glob("C:\\windows\\system32\\some*.txt"))
+        assert len(results) == 1
+
+        results = list(target_win.fs.path("/").glob("sysvol/windows/system32/some*.txt"))
+        assert len(results) == 1

tests/test_helpers_keychain.py → tests/helpers/test_keychain.py

@@ -3,8 +3,7 @@
 import pytest
 
 from dissect.target.helpers import keychain
-
-from ._utils import absolute_path
+from tests._utils import absolute_path
 
 
 @pytest.fixture
@@ -15,7 +14,7 @@ def guarded_keychain():
 
 
 def test_keychain_register_keychain_file(guarded_keychain):
-    keychain_file = Path(absolute_path("data/keychain.csv"))
+    keychain_file = Path(absolute_path("_data/helpers/keychain/keychain.csv"))
 
     keychain.register_keychain_file(keychain_file)
 

tests/test_helpers_regutil.py → tests/helpers/test_regutil.py

@@ -16,14 +16,13 @@
     glob_split,
     has_glob_magic,
 )
-
-from ._utils import absolute_path
+from tests._utils import absolute_path
 
 
 def test_regflex() -> None:
     regflex = RegFlex()
 
-    with open(absolute_path("data/regflex.reg"), "rt") as fh:
+    with open(absolute_path("_data/helpers/regutil/regflex.reg"), "rt") as fh:
         regflex.map_definition(fh)
 
     assert "HKEY_CURRENT_USER" in regflex.hives

tests/test_loaders_asdf.py → tests/loaders/test_asdf.py

@@ -2,12 +2,11 @@
 
 from dissect.target import Target
 from dissect.target.loaders.asdf import AsdfLoader
-
-from ._utils import absolute_path
+from tests._utils import absolute_path
 
 
 def test_asdf_loader_metadata(target_bare: Target):
-    asdf_path = Path(absolute_path("data/loaders/asdf/metadata.asdf"))
+    asdf_path = Path(absolute_path("_data/loaders/asdf/metadata.asdf"))
 
     loader = AsdfLoader(asdf_path)
     loader.map(target_bare)