fix: Implement correct file filter syntax and ignore_rules for FDD

Author: cph-datamasque

datamasque/client/__init__.py

@@ -61,6 +61,8 @@
     FileDiscoveryLocatorResult,
     FileDiscoveryMatch,
     FileDiscoveryResult,
+    FileFilter,
+    FileFilterMatchAgainst,
     FileRulesetGenerationRequest,
     ForeignKeyRef,
     InDataDiscoveryConfig,
@@ -146,6 +148,8 @@
     "FileDiscoveryLocatorResult",
     "FileDiscoveryMatch",
     "FileDiscoveryResult",
+    "FileFilter",
+    "FileFilterMatchAgainst",
     "FileId",
     "FileOrContent",
     "FileRulesetGenerationRequest",

datamasque/client/models/discovery.py

@@ -1,8 +1,9 @@
 """Typed request and response shapes for schema-discovery and ruleset-generation endpoints."""
 
+from enum import Enum
 from typing import Any, Optional, Union
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from datamasque.client.models.connection import ConnectionConfig, ConnectionId, unwrap_connection_id
 from datamasque.client.models.data_selection import HashColumnsTableConfig, Locator, UserSelection
@@ -28,6 +29,7 @@ class InDataDiscoveryConfig(BaseModel):
     row_sample_size: Optional[int] = None
     custom_rules: Optional[list[InDataDiscoveryRule]] = None
     non_sensitive_rules: Optional[list[InDataDiscoveryRule]] = None
+    ignore_rules: Optional[list[InDataDiscoveryRule]] = None
     force: Optional[bool] = None
 
 
@@ -86,6 +88,35 @@ def _unwrap_connection(cls, value: Any) -> Any:
         return unwrap_connection_id(value)
 
 
+class FileFilterMatchAgainst(str, Enum):
+    """Which part of a file's path an `include`/`skip` filter is matched against."""
+
+    path = "path"
+    filename = "filename"
+
+
+class FileFilter(BaseModel):
+    """
+    A single `include` or `skip` filter for file data discovery.
+
+    Exactly one of `glob` or `regex` must be set.
+    `match_against` selects whether the pattern is applied to the full path or just the filename
+    (the server defaults to the full path when omitted).
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    glob: Optional[str] = None
+    regex: Optional[str] = None
+    match_against: Optional[FileFilterMatchAgainst] = None
+
+    @model_validator(mode="after")
+    def _check_glob_xor_regex(self) -> "FileFilter":
+        if bool(self.glob) == bool(self.regex):
+            raise ValueError("A `FileFilter` must set exactly one of `glob` or `regex`.")
+        return self
+
+
 class FileDataDiscoveryOptions(BaseModel):
     """Run options nested under `FileDataDiscoveryRequest.options`."""
 
@@ -117,8 +148,8 @@ class FileDataDiscoveryRequest(BaseModel):
     disable_global_ignored_keywords: Optional[bool] = None
     in_data_discovery: Optional[InDataDiscoveryConfig] = None
     recurse: Optional[bool] = None
-    include: Optional[list[str]] = None
-    skip: Optional[list[str]] = None
+    include: Optional[list[FileFilter]] = None
+    skip: Optional[list[FileFilter]] = None
     encoding: Optional[str] = None
     workers: Optional[int] = None
 

tests/test_discovery.py

@@ -6,14 +6,18 @@
 
 import pytest
 import requests_mock
+from pydantic import ValidationError
 
 from datamasque.client import (
     DataMasqueClient,
     DiscoveryConfig,
     DiscoveryConfigId,
     FileDataDiscoveryOptions,
     FileDataDiscoveryRequest,
+    FileFilter,
+    FileFilterMatchAgainst,
     FileRulesetGenerationRequest,
+    InDataDiscoveryConfig,
     RulesetGenerationRequest,
     RunId,
     SchemaDiscoveryPage,
@@ -808,8 +812,8 @@ def test_start_file_data_discovery_run_full(client):
         disable_global_ignored_keywords=False,
         in_data_discovery={"enabled": True, "row_sample_size": 50},
         recurse=True,
-        include=["*.csv"],
-        skip=["**/tmp/**"],
+        include=[{"glob": "*.csv"}],
+        skip=[{"regex": r".*/tmp/.*", "match_against": "path"}],
         encoding="utf-8",
         workers=4,
     )
@@ -833,13 +837,41 @@ def test_start_file_data_discovery_run_full(client):
         "disable_global_ignored_keywords": False,
         "in_data_discovery": {"enabled": True, "row_sample_size": 50},
         "recurse": True,
-        "include": ["*.csv"],
-        "skip": ["**/tmp/**"],
+        "include": [{"glob": "*.csv"}],
+        "skip": [{"regex": r".*/tmp/.*", "match_against": "path"}],
         "encoding": "utf-8",
         "workers": 4,
     }
 
 
+def test_file_filter_requires_exactly_one_of_glob_or_regex():
+    """A `FileFilter` with neither, or both, of `glob`/`regex` is rejected."""
+    FileFilter(glob="*.csv")
+    FileFilter(regex=r".*\.csv")
+    FileFilter(glob="*.csv", match_against=FileFilterMatchAgainst.filename)
+
+    with pytest.raises(ValidationError, match="exactly one of `glob` or `regex`"):
+        FileFilter()
+
+    with pytest.raises(ValidationError, match="exactly one of `glob` or `regex`"):
+        FileFilter(glob="*.csv", regex=r".*\.csv")
+
+
+def test_file_data_discovery_ignore_rules_serialize():
+    """`in_data_discovery.ignore_rules` round-trips into the wire payload."""
+    req = FileDataDiscoveryRequest(
+        connection="conn-1",
+        in_data_discovery=InDataDiscoveryConfig(
+            enabled=True,
+            custom_rules=[{"name": "cc", "pattern": r"^1234"}],
+            non_sensitive_rules=[{"pattern": r"^5555"}],
+            ignore_rules=[{"pattern": r"^4321"}],
+        ),
+    )
+    dumped = req.model_dump(exclude_none=True, mode="json")
+    assert dumped["in_data_discovery"]["ignore_rules"] == [{"pattern": r"^4321"}]
+
+
 def test_start_file_data_discovery_run_raises_on_non_201(client):
     with requests_mock.Mocker() as m:
         m.post(