Compatibility with cstruct v4 (fox-it#717)

.github/workflows/dissect-ci.yml

@@ -11,6 +11,8 @@ on:
 jobs:
   ci:
     uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-template.yml@main
+    with:
+      deb-packages: 'liblzo2-dev'
     secrets: inherit
 
   publish:

.github/workflows/on-demand-test.yml

@@ -16,6 +16,7 @@ jobs:
             python-include: "pypy3.9"
             tox-env: "pypy39"
     steps:
+      - run: sudo apt-get install -qq liblzo2-dev
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0

dissect/target/helpers/protobuf.py

@@ -3,30 +3,26 @@
 from typing import Any, BinaryIO
 
 from dissect.cstruct.types.base import BaseType
-from dissect.cstruct.types.bytesinteger import BytesInteger
 
 
-class ProtobufVarint(BytesInteger):
+class ProtobufVarint(BaseType):
     """Implements a protobuf integer type for dissect.cstruct that can span a variable amount of bytes.
 
-    Mainly follows the cstruct BytesInteger implementation with minor tweaks
-    to support protobuf's msb varint implementation.
+    Supports protobuf's msb varint implementation.
 
     Resources:
         - https://protobuf.dev/programming-guides/encoding/
         - https://github.com/protocolbuffers/protobuf/blob/main/python/google/protobuf/internal/decoder.py
     """
 
-    def _read(self, stream: BinaryIO, context: dict[str, Any] = None) -> int:
+    @classmethod
+    def _read(cls, stream: BinaryIO, context: dict[str, Any] = None) -> int:
         return decode_varint(stream)
 
-    def _write(self, stream: BinaryIO, data: int) -> int:
+    @classmethod
+    def _write(cls, stream: BinaryIO, data: int) -> int:
         return stream.write(encode_varint(data))
 
-    _read_array = BaseType._read_array
-
-    _write_array = BaseType._write_array
-
 
 def decode_varint(stream: BinaryIO) -> int:
     """Reads a varint from the provided buffer stream.

dissect/target/helpers/ssh.py

@@ -1,9 +1,9 @@
 import base64
 import binascii
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
-c_rfc4716_def = """
+rfc4716_def = """
 struct ssh_string {
     uint32 length;
     char value[length];
@@ -23,8 +23,7 @@
 }
 """
 
-c_rfc4716 = cstruct.cstruct(endian=">")
-c_rfc4716.load(c_rfc4716_def)
+c_rfc4716 = cstruct(endian=">").load(rfc4716_def)
 
 RFC4716_MARKER_START = b"-----BEGIN OPENSSH PRIVATE KEY-----"
 RFC4716_MARKER_END = b"-----END OPENSSH PRIVATE KEY-----"

dissect/target/plugins/apps/av/trendmicro.py

@@ -1,6 +1,6 @@
 from typing import Iterator
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util.ts import from_unix
 
 from dissect.target import Target
@@ -47,8 +47,7 @@
     char      _pad3[10];
 };
 """
-c_pfwlog = cstruct.cstruct()
-c_pfwlog.load(pfwlog_def)
+c_pfwlog = cstruct().load(pfwlog_def)
 
 
 class TrendMicroPlugin(Plugin):

dissect/target/plugins/apps/container/docker.py

@@ -88,7 +88,7 @@
 """
 
 c_local = cstruct(endian=">")
-c_local.addtype("varint", ProtobufVarint(c_local, "varint", size=None, signed=False, alignment=1))
+c_local.add_custom_type("varint", ProtobufVarint, size=None, alignment=1, signed=False)
 c_local.load(local_def, compiled=False)
 
 RE_DOCKER_NS = re.compile(r"\.(?P<nanoseconds>\d{7,})(?P<postfix>Z|\+\d{2}:\d{2})")

dissect/target/plugins/os/unix/locate/gnulocate.py

@@ -26,8 +26,7 @@
     ],
 )
 
-c_gnulocate = cstruct()
-c_gnulocate.load(gnulocate_def)
+c_gnulocate = cstruct().load(gnulocate_def)
 
 
 class GNULocateFile:

dissect/target/plugins/os/unix/locate/mlocate.py

@@ -20,10 +20,10 @@
     int32 conf_size;
     int8 version;                            /* file format version */
     int8 require_visibility;
-    int8 pad[2];                             /* 32-bit total alignment */
+    int8 pad0[2];                             /* 32-bit total alignment */
     char root_database;
     char config_block[conf_size];
-    int8 pad;
+    int8 pad1;
 };
 
 enum DBE_TYPE: uint8 {                       /* database entry type */
@@ -68,8 +68,7 @@ class MLocate:
     ],
 )
 
-c_mlocate = cstruct(endian=">")
-c_mlocate.load(mlocate_def)
+c_mlocate = cstruct(endian=">").load(mlocate_def)
 
 
 class MLocateFile:

dissect/target/plugins/os/unix/locate/plocate.py

@@ -65,8 +65,7 @@
     ],
 )
 
-c_plocate = cstruct()
-c_plocate.load(plocate_def)
+c_plocate = cstruct().load(plocate_def)
 
 
 class PLocateFile:

dissect/target/plugins/os/unix/log/atop.py

@@ -2,7 +2,7 @@
 from io import BytesIO
 from typing import BinaryIO, Iterator
 
-from dissect.cstruct import Instance, cstruct
+from dissect.cstruct import cstruct
 
 from dissect.target.exceptions import UnsupportedPluginError
 from dissect.target.helpers.record import TargetRecordDescriptor
@@ -178,8 +178,7 @@
 };
 """  # noqa: E501
 
-c_atop = cstruct()
-c_atop.load(atop_def)
+c_atop = cstruct().load(atop_def)
 c_atop.load(atop_tstat_def, align=True)
 
 AtopRecord = TargetRecordDescriptor(
@@ -226,7 +225,7 @@ def __init__(self, fh: BinaryIO):
         self.header = c_atop.rawheader(self.fh)
         self.version = self.version()
 
-    def __iter__(self) -> Iterator[Instance]:
+    def __iter__(self) -> Iterator[c_atop.tstat]:
         while True:
             try:
                 record = c_atop.rawrecord(self.fh)

dissect/target/plugins/os/unix/log/journal.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import lzma
 from typing import BinaryIO, Callable, Iterator
 
 import zstandard
-from dissect.cstruct import Instance, cstruct
+from dissect.cstruct import cstruct
 from dissect.util import ts
 from dissect.util.compression import lz4
 
@@ -252,8 +254,7 @@
 };
 """  # noqa: E501
 
-c_journal = cstruct()
-c_journal.load(journal_def)
+c_journal = cstruct().load(journal_def)
 
 
 def get_optional(value: str, to_type: Callable):
@@ -314,7 +315,7 @@ def decode_value(self, value: bytes) -> tuple[str, str]:
 
         return key, value
 
-    def __iter__(self) -> Iterator[Instance]:
+    def __iter__(self) -> Iterator[dict[str, int | str]]:
         "Iterate over the entry objects to read payloads."
 
         for offset in self.entry_object_offsets():

dissect/target/plugins/os/unix/log/lastlog.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util import ts
 
 from dissect.target.exceptions import FileNotFoundError, UnsupportedPluginError
@@ -36,8 +36,7 @@
 };
 """
 
-c_lastlog = cstruct.cstruct()
-c_lastlog.load(lastlog_def)
+c_lastlog = cstruct().load(lastlog_def)
 
 
 class LastLogFile:

dissect/target/plugins/os/unix/log/utmp.py

@@ -39,14 +39,14 @@
     ],
 )
 
-c_utmp = """
+utmp_def = """
 #define UT_LINESIZE     32
 #define UT_NAMESIZE     32
 #define UT_HOSTSIZE     256
 
 typedef uint32 pid_t;
 
-enum Type : char {
+enum Type : uint8_t {
     EMPTY           = 0x0,
     RUN_LVL         = 0x1,
     BOOT_TIME       = 0x2,
@@ -84,8 +84,7 @@
 };
 """  # noqa: E501
 
-utmp = cstruct()
-utmp.load(c_utmp)
+c_utmp = cstruct().load(utmp_def)
 
 UTMP_ENTRY = namedtuple(
     "UTMPRecord",
@@ -122,11 +121,11 @@ def __iter__(self):
 
         while True:
             try:
-                entry = utmp.entry(byte_stream)
+                entry = c_utmp.entry(byte_stream)
 
                 r_type = ""
-                if entry.ut_type in utmp.Type.reverse:
-                    r_type = utmp.Type.reverse[entry.ut_type]
+                if entry.ut_type in c_utmp.Type:
+                    r_type = c_utmp.Type(entry.ut_type).name
 
                 ut_host = entry.ut_host.decode(errors="surrogateescape").strip("\x00")
                 ut_addr = None

dissect/target/plugins/os/windows/adpolicy.py

@@ -1,7 +1,7 @@
 from struct import unpack
 
 from defusedxml import ElementTree
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.regf.c_regf import (
     REG_BINARY,
     REG_DWORD,
@@ -18,14 +18,13 @@
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.plugin import Plugin, export
 
-c_def = """
+policy_def = """
 struct registry_policy_header {
     uint32   signature;
     uint32   version;
 };
 """
-c_adpolicy = cstruct.cstruct()
-c_adpolicy.load(c_def)
+c_adpolicy = cstruct().load(policy_def)
 
 ADPolicyRecord = TargetRecordDescriptor(
     "windows/adpolicy",

dissect/target/plugins/os/windows/credhist.py

@@ -53,8 +53,7 @@
 };
 """
 
-c_credhist = cstruct()
-c_credhist.load(credhist_def)
+c_credhist = cstruct().load(credhist_def)
 
 
 @dataclass

dissect/target/plugins/os/windows/datetime.py

@@ -3,7 +3,7 @@
 from datetime import datetime, timedelta, timezone, tzinfo
 from typing import Dict, Tuple
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 from dissect.target.exceptions import (
     RegistryError,
@@ -34,8 +34,7 @@
     SYSTEMTIME DaylightDate;
 } REG_TZI_FORMAT;
 """
-c_tz = cstruct.cstruct()
-c_tz.load(tz_def)
+c_tz = cstruct().load(tz_def)
 
 
 # Althoug calendar.SUNDAY is only officially documented since Python 3.10, it
@@ -63,7 +62,7 @@
 HOUR = timedelta(hours=1)
 
 
-def parse_systemtime_transition(systemtime: cstruct.Instance, year: int) -> datetime:
+def parse_systemtime_transition(systemtime: c_tz._SYSTEMTIME, year: int) -> datetime:
     """Return the transition datetime for a given year using the SYSTEMTIME of a STD or DST transition date.
 
     The SYSTEMTIME date of a TZI structure needs to be used to calculate the actual date for a given year.

dissect/target/plugins/os/windows/defender.py

@@ -237,8 +237,7 @@
 };
 """
 
-c_defender = cstruct()
-c_defender.load(defender_def)
+c_defender = cstruct().load(defender_def)
 
 STREAM_ID = c_defender.STREAM_ID
 STREAM_ATTRIBUTES = c_defender.STREAM_ATTRIBUTES
@@ -381,7 +380,7 @@ def _add_field(self, field: Structure):
             self.last_access_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
         elif field.Identifier == FIELD_IDENTIFIER.LastWriteTime:
             self.last_write_time = ts.wintimestamp(int.from_bytes(field.Data, "little"))
-        elif field.Identifier not in FIELD_IDENTIFIER.values.values():
+        elif field.Identifier not in FIELD_IDENTIFIER:
             self.unknown_fields.append(field)
 
 
@@ -526,7 +525,7 @@ def recover(self, output_dir: Path) -> None:
                     subdir = resource.resource_id[0:2]
                     resourcedata_location = resourcedata_directory.joinpath(subdir).joinpath(resource.resource_id)
                     if not resourcedata_location.exists():
-                        self.target.log.warning(f"Could not find a ResourceData file for {entry.resource_id}.")
+                        self.target.log.warning(f"Could not find a ResourceData file for {resource.resource_id}.")
                         continue
                     if not resourcedata_location.is_file():
                         self.target.log.warning(f"{resourcedata_location} is not a file!")

dissect/target/plugins/os/windows/dpapi/blob.py

@@ -36,8 +36,7 @@
 };
 """
 
-c_blob = cstruct()
-c_blob.load(blob_def)
+c_blob = cstruct().load(blob_def)
 
 
 class Blob:

dissect/target/plugins/os/windows/dpapi/master_key.py

@@ -29,7 +29,7 @@
     DWORD   accessCheckLen;
     char    guid[16];
     char    encryptedSecret[secretLen];
-    char    accessCheckLen[accessCheckLen];
+    char    accessCheck[accessCheckLen];
 };
 
 struct CredHist {
@@ -66,8 +66,7 @@
     QWORD   qwDomainKeySize;
 };
 """
-c_master_key = cstruct()
-c_master_key.load(master_key_def)
+c_master_key = cstruct().load(master_key_def)
 
 
 class MasterKey:

dissect/target/plugins/os/windows/notifications.py

@@ -91,8 +91,7 @@
 } Chunk;                               // size: 0x23810
 """
 
-c_appdb = cstruct(endian="<")
-c_appdb.load(appdb_def)
+c_appdb = cstruct(endian="<").load(appdb_def)
 
 APPDB_MAGIC = b"DNPW"
 NUM_APPDB_CHUNKS = 256