Merge branch 'main' into handle-missing-pv-error

.github/pull_request_template.md

@@ -0,0 +1,24 @@
+<!--
+Thank you for submitting a Pull Request. Please:
+* Read our commit style guide:
+    Commit messages should adhere to the following points:
+    * Separate subject from body with a blank line
+    * Limit the subject line to 50 characters as much as possible
+    * Capitalize the subject line
+    * Do not end the subject line with a period
+    * Use the imperative mood in the subject line
+    * The verb should represent what was accomplished (Create, Add, Fix etc)
+    * Wrap the body at 72 characters
+    * Use the body to explain the what and why vs. the how
+  For an example, look at the following link:
+  https://docs.dissect.tools/en/latest/contributing/style-guide.html#example-commit-message
+
+* Include a description of the proposed changes and how to test them.
+
+* After creation, associate the PR with an issue, under the development section.
+  Or use closing keywords in the body during creation:
+  E.G:
+  * close(|s|d) #<nr>
+  * fix(|es|ed) #<nr>
+  * resolve(|s|d) #<nr>
+-->

.github/workflows/dissect-ci.yml

@@ -3,6 +3,8 @@ on:
   push:
     branches:
       - main
+    tags:
+      - '*'
   pull_request:
   workflow_dispatch:
 
@@ -19,7 +21,7 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: packages
           path: dist/
@@ -30,5 +32,6 @@ jobs:
   trigger-tests:
     needs: [publish]
     uses: fox-it/dissect-workflow-templates/.github/workflows/dissect-ci-demand-test-template.yml@main
+    secrets: inherit
     with:
       on-demand-test: 'dissect.target'

dissect/volume/ddf/c_ddf.py

@@ -141,12 +141,12 @@
 struct Virtual_Disk_Entry {
     char    VD_GUID[24];
     uint16  VD_Number;
-    char    Reserved[2];
+    char    Reserved1[2];
     uint32  VD_Type;
     uint8   VD_State;
     uint8   Init_State;
     uint8   Partially_Optimal_Drive_Failures_Remaining;     /* DDF 2.0 */
-    char    Reserved[13];
+    char    Reserved2[13];
     char    VD_Name[16];
 };
 
@@ -322,7 +322,6 @@
     char    Reserved[16];
 };
 """
-c_ddf = cstruct(endian=">")
-c_ddf.load(ddf_def)
+c_ddf = cstruct(endian=">").load(ddf_def)
 
 DEFAULT_SECTOR_SIZE = 512

dissect/volume/ddf/ddf.py

@@ -3,7 +3,6 @@
 import io
 from typing import BinaryIO, Union
 
-from dissect.cstruct.types import Instance, Structure
 from dissect.util import ts
 
 from dissect.volume.ddf.c_ddf import DEFAULT_SECTOR_SIZE, c_ddf
@@ -251,7 +250,13 @@ def __repr__(self) -> str:
         return f"<VirtualDiskConfigurationRecord guid={self.guid}>"
 
 
-def _read_section_header(fh: BinaryIO, structure: Structure, signature: int) -> Instance:
+def _read_section_header(
+    fh: BinaryIO,
+    structure: type[
+        c_ddf.Physical_Disk_Records | c_ddf.Virtual_Disk_Records | c_ddf.Controller_Data | c_ddf.Physical_Disk_Data
+    ],
+    signature: int,
+) -> c_ddf.Physical_Disk_Records | c_ddf.Virtual_Disk_Records | c_ddf.Controller_Data | c_ddf.Physical_Disk_Data:
     obj = structure(fh)
     if obj.Signature != signature:
         raise DDFError(f"Invalid {structure.name} header. Signature: {obj.Signature:#010x}, expected {signature:#x}")

dissect/volume/disk/partition.py

@@ -1,7 +1,7 @@
 from typing import Any, BinaryIO, Optional, Union
 from uuid import UUID
 
-from dissect.cstruct import Instance
+from dissect.cstruct import Structure
 from dissect.util.stream import RangeStream
 
 PARTITION_TYPES = {
@@ -185,7 +185,7 @@ def __init__(
         flags: Optional[int] = None,
         guid: Optional[UUID] = None,
         vtype_str: Optional[str] = None,
-        raw: Instance = None,
+        raw: Structure = None,
     ):
         self.disk = disk
         self.number = number

dissect/volume/disk/schemes/apm.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO, Iterator
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 from dissect.volume.disk.partition import Partition
 from dissect.volume.exceptions import DiskError
@@ -29,9 +29,7 @@
 };
 """
 
-c_apm = cstruct.cstruct()
-c_apm.load(apm_def)
-c_apm.endian = ">"
+c_apm = cstruct(endian=">").load(apm_def)
 
 
 class APM:

dissect/volume/disk/schemes/bsd.py

@@ -5,7 +5,7 @@
 from typing import BinaryIO, Iterator
 from uuid import UUID
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 from dissect.volume.disk.partition import Partition
 from dissect.volume.exceptions import DiskError
@@ -215,8 +215,7 @@
 #define D_CHAIN             0x10                /* can do back-back transfers */
 """
 
-c_bsd = cstruct.cstruct()
-c_bsd.load(bsd_def)
+c_bsd = cstruct().load(bsd_def)
 
 DTYPE_NAMES = {
     0: "unknown",

dissect/volume/disk/schemes/gpt.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO, Iterator
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 from dissect.volume.disk.partition import Partition
 from dissect.volume.disk.schemes.mbr import MBR
@@ -43,8 +43,7 @@
 // 56 (0x38)    72 bytes     Partition name (36 UTF-16LE code units)
 """
 
-c_gpt = cstruct.cstruct()
-c_gpt.load(gpt_def)
+c_gpt = cstruct().load(gpt_def)
 
 
 class GPT:
@@ -94,6 +93,7 @@ def _partitions(self) -> Iterator[Partition]:
 
                     name = (
                         partition.name.decode("utf-16-le", "ignore")
+                        .split("\x00")[0]
                         .strip("\x00")
                         .strip("\uffff")  # a non-character in UTF-16
                     )

dissect/volume/disk/schemes/mbr.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO, Iterator, Optional
 
-from dissect.cstruct import Instance, cstruct
+from dissect.cstruct import cstruct
 
 from dissect.volume.disk.partition import Partition
 from dissect.volume.exceptions import DiskError
@@ -26,8 +26,7 @@
 } mbr;
 """
 
-c_mbr = cstruct()
-c_mbr.load(mbr_def)
+c_mbr = cstruct().load(mbr_def)
 
 
 class MBR:
@@ -53,7 +52,7 @@ def __init__(self, fh: BinaryIO, sector_size: int = 512):
         self.partitions: list[Partition] = list(self._partitions(self.mbr, self.offset))
 
     def _partitions(
-        self, mbr: Instance, offset: int, num_start: int = 0, ebr_offset: Optional[int] = None
+        self, mbr: c_mbr.mbr_s, offset: int, num_start: int = 0, ebr_offset: Optional[int] = None
     ) -> Iterator[Partition]:
         for num, partition in enumerate(mbr.part):
             if partition.type == 0x00:

dissect/volume/dm/c_dm.py

@@ -87,7 +87,6 @@
 };
 """
 
-c_dm = cstruct()
-c_dm.load(dm_def)
+c_dm = cstruct().load(dm_def)
 
 SECTOR_SIZE = 512

dissect/volume/ldm.py

@@ -5,7 +5,7 @@
 import logging
 import os
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 
 log = logging.getLogger(__name__)
 log.setLevel(os.getenv("DISSECT_LOG_LDM", "CRITICAL"))
@@ -246,5 +246,4 @@
 };
 """
 
-c_ldm = cstruct.cstruct(endian=">")
-c_ldm.load(ldm_def)
+c_ldm = cstruct(endian=">").load(ldm_def)

dissect/volume/lvm/c_lvm2.py

@@ -38,8 +38,7 @@
 #define RAW_LOCN_IGNORED    0x00000001
 """
 
-c_lvm = cstruct()
-c_lvm.load(lvm_def)
+c_lvm = cstruct().load(lvm_def)
 
 SECTOR_SIZE = 512
 

dissect/volume/lvm/physical.py

@@ -1,10 +1,11 @@
+from __future__ import annotations
+
 import ast
 import re
 from bisect import bisect_right
 from functools import cached_property
 from typing import Any, BinaryIO, Iterator, Optional
 
-from dissect.cstruct import Instance, Structure
 from dissect.util.stream import RunlistStream
 
 from dissect.volume.exceptions import LVM2Error
@@ -92,7 +93,9 @@ def open(self) -> BinaryIO:
         return RunlistStream(self.fh, runlist, self.size, SECTOR_SIZE)
 
 
-def _read_descriptors(fh: BinaryIO, ctype: Structure) -> list[Instance]:
+def _read_descriptors(
+    fh: BinaryIO, ctype: type[c_lvm.disk_locn | c_lvm.raw_locn]
+) -> list[c_lvm.disk_locn | c_lvm.raw_locn]:
     descriptors = []
     while True:
         desc = ctype(fh)

dissect/volume/md/c_md.py

@@ -279,7 +279,6 @@
 #define ALGORITHM_PARITY_N_6            ALGORITHM_PARITY_N
 """  # noqa: E501
 
-c_md = cstruct()
-c_md.load(md_def)
+c_md = cstruct().load(md_def)
 
 SECTOR_SIZE = 512

dissect/volume/vss.py

@@ -5,7 +5,7 @@
 import uuid
 from collections import defaultdict
 
-from dissect import cstruct
+from dissect.cstruct import cstruct
 from dissect.util.stream import AlignedStream
 
 from dissect.volume.exceptions import Error
@@ -144,8 +144,7 @@
 };
 """
 
-c_vss = cstruct.cstruct()
-c_vss.load(vss_def)
+c_vss = cstruct().load(vss_def)
 
 RECORD_TYPE = c_vss.RECORD_TYPE
 BLOCK_FLAG = c_vss.BLOCK_FLAG

pyproject.toml

@@ -25,8 +25,8 @@ classifiers = [
   "Topic :: Utilities",
 ]
 dependencies = [
-    "dissect.cstruct>=3.0.dev,<4.0.dev",
-    "dissect.util>=3.0.dev,<4.0.dev",
+    "dissect.cstruct>=4,<5",
+    "dissect.util>3,<4",
 ]
 dynamic = ["version"]
 
@@ -35,6 +35,12 @@ homepage = "https://dissect.tools"
 documentation = "https://docs.dissect.tools/en/latest/projects/dissect.volume"
 repository = "https://github.com/fox-it/dissect.volume"
 
+[project.optional-dependencies]
+dev = [
+    "dissect.cstruct>=4.0.dev,<5.0.dev",
+    "dissect.util>=3.0.dev,<4.0.dev",
+]
+
 [tool.black]
 line-length = 120
 

tox.ini

@@ -4,13 +4,15 @@ envlist = lint, py3, pypy3
 # requires if they are not available on the host system. This requires the
 # locally installed tox to have a minimum version 3.3.0. This means the names
 # of the configuration options are still according to the tox 3.x syntax.
-minversion = 4.2.4
-# This version of virtualenv will install setuptools version 65.5.0 and pip
-# 22.3. These versions fully support python projects defined only through a
-# pyproject.toml file (PEP-517/PEP-518/PEP-621)
-requires = virtualenv>=20.16.6
+minversion = 4.4.3
+# This version of virtualenv will install setuptools version 68.2.2 and pip
+# 23.3.1. These versions fully support python projects defined only through a
+# pyproject.toml file (PEP-517/PEP-518/PEP-621). This pip version also support
+# the proper version resolving with (sub-)dependencies defining dev extra's.
+requires = virtualenv>=20.24.6
 
 [testenv]
+extras = dev
 deps =
     pytest
     pytest-cov