Merge pull request #21 from datamasque/epic/discovery-config

Add support for discovery configurations

Author: cph-datamasque

.github/workflows/release-testpypi.yml

@@ -2,13 +2,26 @@ name: Release (TestPyPI)
 
 on:
   workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Branch, tag, or commit SHA to build from'
+        required: true
+        default: 'main'
+      dev_version:
+        description: >-
+          Optional PEP 440 pre/dev version override (e.g. 1.1.0.dev1, 1.1.0a2, 1.1.0rc1).
+          Leave blank to use the version in pyproject.toml as-is.
+        required: false
+        default: ''
 
 jobs:
   build:
     name: Build sdist and wheel
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
 
       - name: Set up uv
         uses: astral-sh/setup-uv@v5
@@ -18,11 +31,37 @@ jobs:
       - name: Set up Python
         run: uv python install 3.12
 
+      - name: Apply dev_version override
+        if: inputs.dev_version != ''
+        # Pass the input through the environment, never interpolated into the script body:
+        # `${{ }}` is expanded into the script text before the shell runs, so splicing it inline
+        # would let a crafted dev_version inject shell commands before the validation below could reject it.
+        env:
+          DEV_VERSION: ${{ inputs.dev_version }}
+        run: |
+          # Reject anything that isn't a pre-release / dev version — final releases must go through release.yml.
+          if ! printf '%s' "${DEV_VERSION}" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(a|b|rc|\.dev)[0-9]+$'; then
+            echo "::error::dev_version '${DEV_VERSION}' is not a PEP 440 pre/dev version (must end in aN, bN, rcN, or .devN)"
+            exit 1
+          fi
+          uv run python -c "
+          import os, re, pathlib
+          version = os.environ['DEV_VERSION']
+          path = pathlib.Path('pyproject.toml')
+          text = path.read_text()
+          new = re.sub(r'^version\s*=\s*\".*\"', f'version = \"{version}\"', text, count=1, flags=re.M)
+          if new == text:
+              raise SystemExit('Failed to locate version line in pyproject.toml')
+          path.write_text(new)
+          "
+
       - name: Show package version
+        env:
+          REF: ${{ inputs.ref }}
         run: |
           VERSION="$(uv run python -c 'import tomllib; print(tomllib.loads(open("pyproject.toml","rb").read().decode())["project"]["version"])')"
-          echo "Publishing version: ${VERSION}"
-          echo "::notice title=TestPyPI version::${VERSION}"
+          echo "Publishing version: ${VERSION} (from ref ${REF})"
+          echo "::notice title=TestPyPI version::${VERSION} from ${REF}"
 
       - name: Build
         run: uv build

HISTORY.rst

@@ -2,6 +2,29 @@
 History
 =======
 
+1.1.0 (2026-06-24)
+------------------
+
+* Added discovery configuration models and management APIs.
+* Added schema-discovery and file-data-discovery APIs that take a saved discovery configuration
+  (``start_schema_discovery_run_from_config`` / ``start_file_data_discovery_run_from_config``).
+  Adoption is recommended; the older APIs that take individual options will be deprecated in a future release.
+* Corrected the file-data-discovery ``include``/``skip`` filter syntax and added ``ignore_rules`` support.
+* Added ``InvalidDiscoveryConfigError`` and ``DiscoveryConfigNotFoundError``,
+  raised when a discovery run can't start due to an unusable or missing discovery config.
+* Added ``get_discovery_run_config_snapshot_yaml`` to retrieve the discovery-config YAML
+  that was effective at the start of a given discovery run.
+* Added ``is_user_subscribed`` to ``MaskingRunRequest`` to subscribe the requesting user to a run's email notifications.
+* Added ``auto_pull`` / ``auto_pull_branch`` to ``MaskingRunOptions``
+  to refresh the run's ruleset from git before starting.
+* Added ``validation_error`` (and ``validation_error_type`` for rulesets) to ``Ruleset`` and ``RulesetLibrary``.
+* Exposed git provenance on ``Ruleset`` and ``RulesetLibrary`` as a nested ``git`` field (``GitSnapshot``).
+* Read-only fields (``id``, ``is_valid``, ``validation_error``, etc.)
+  are no longer echoed back in ``Ruleset`` / ``RulesetLibrary`` create/update request bodies.
+* Fixed ``SslZipFile`` uploads to send the required ``database_type=mysql`` form field.
+* **Breaking:** ``delete_ruleset_by_name_if_exists`` now requires a ``ruleset_type`` argument,
+  since ruleset names are unique only per type.
+
 1.0.5 (2026-06-18)
 ------------------
 

datamasque/client/__init__.py

@@ -17,8 +17,10 @@
     DataMasqueNotReadyError,
     DataMasqueTransportError,
     DataMasqueUserError,
+    DiscoveryConfigNotFoundError,
     FailedToStartError,
     IfmAuthError,
+    InvalidDiscoveryConfigError,
     InvalidLibraryError,
     InvalidRulesetError,
     RunNotCancellableError,
@@ -54,22 +56,29 @@
 from datamasque.client.models.discovery import (
     ConstraintColumns,
     DiscoveryMatch,
+    FileDataDiscoveryFromConfigRequest,
+    FileDataDiscoveryOptions,
+    FileDataDiscoveryRequest,
     FileDiscoveryFile,
     FileDiscoveryLocatorResult,
     FileDiscoveryMatch,
     FileDiscoveryResult,
+    FileFilter,
+    FileFilterMatchAgainst,
     FileRulesetGenerationRequest,
     ForeignKeyRef,
     InDataDiscoveryConfig,
     InDataDiscoveryRule,
     ReferencingForeignKey,
     RulesetGenerationRequest,
     SchemaDiscoveryColumn,
+    SchemaDiscoveryFromConfigRequest,
     SchemaDiscoveryPage,
     SchemaDiscoveryRequest,
     SchemaDiscoveryResult,
     TableConstraints,
 )
+from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigId, DiscoveryConfigType
 from datamasque.client.models.dm_instance import DataMasqueInstanceConfig
 from datamasque.client.models.files import (
     DataMasqueFile,
@@ -79,6 +88,7 @@
     SnowflakeKeyFile,
     SslZipFile,
 )
+from datamasque.client.models.git import GitSnapshot
 from datamasque.client.models.ifm import (
     DataMasqueIfmInstanceConfig,
     IfmLog,
@@ -104,7 +114,12 @@
     RunInfo,
     UnfinishedRun,
 )
-from datamasque.client.models.status import AsyncRulesetGenerationTaskStatus, MaskingRunStatus, ValidationStatus
+from datamasque.client.models.status import (
+    AsyncRulesetGenerationTaskStatus,
+    MaskingRunStatus,
+    ValidationErrorType,
+    ValidationStatus,
+)
 from datamasque.client.models.user import User, UserId, UserRole
 
 __version__ = version("datamasque-python")
@@ -130,18 +145,28 @@
     "DatabaseConnectionConfig",
     "DatabaseType",
     "DatabricksConnectionConfig",
+    "DiscoveryConfig",
+    "DiscoveryConfigId",
+    "DiscoveryConfigNotFoundError",
+    "DiscoveryConfigType",
     "DiscoveryMatch",
     "DynamoConnectionConfig",
     "FailedToStartError",
     "FileConnectionConfig",
+    "FileDataDiscoveryFromConfigRequest",
+    "FileDataDiscoveryOptions",
+    "FileDataDiscoveryRequest",
     "FileDiscoveryFile",
     "FileDiscoveryLocatorResult",
     "FileDiscoveryMatch",
     "FileDiscoveryResult",
+    "FileFilter",
+    "FileFilterMatchAgainst",
     "FileId",
     "FileOrContent",
     "FileRulesetGenerationRequest",
     "ForeignKeyRef",
+    "GitSnapshot",
     "HashColumnsTableConfig",
     "IfmAuthError",
     "IfmLog",
@@ -151,6 +176,7 @@
     "IfmTokenInfo",
     "InDataDiscoveryConfig",
     "InDataDiscoveryRule",
+    "InvalidDiscoveryConfigError",
     "InvalidLibraryError",
     "InvalidRulesetError",
     "JsonPath",
@@ -182,6 +208,7 @@
     "RunNotCancellableError",
     "S3ConnectionConfig",
     "SchemaDiscoveryColumn",
+    "SchemaDiscoveryFromConfigRequest",
     "SchemaDiscoveryPage",
     "SchemaDiscoveryRequest",
     "SchemaDiscoveryResult",
@@ -202,5 +229,6 @@
     "UserId",
     "UserRole",
     "UserSelection",
+    "ValidationErrorType",
     "ValidationStatus",
 ]