feat: add support for data integrity/protection (#81)

Author: bd-mkt

docs/CONFIGURATION.md

@@ -232,6 +232,55 @@ Subdirectory for embedding vectors.
 
 **Default:** `embeddings`
 
+#### FILE_PROTECTION_LEVEL
+
+Protection level for file-stored artifacts. Controls integrity checking and encryption.
+
+**Default:** `none`
+
+**Options:**
+
+- `none` - No protection (current default behavior)
+- `hash` - SHA-512 hash verification: writes a `.hash` sidecar file alongside each artifact
+- `hmac` - HMAC-SHA-512 verification: writes a `.hmac` sidecar file using a keyed hash (requires `FILE_SECRET`)
+- `encrypt` - Fernet encryption: encrypts artifact bytes at rest using authenticated encryption (requires `FILE_SECRET`)
+
+**Example:**
+
+```bash
+FILE_PROTECTION_LEVEL=encrypt
+FILE_SECRET=my-strong-secret-key
+```
+
+**Notes:**
+
+- Only applies when `FILE_STORE_TARGET=fs` (filesystem storage)
+- `hmac` and `encrypt` modes require `FILE_SECRET` to be set
+- A single `FILE_SECRET` is used for both HMAC and encryption; purpose-specific keys are derived internally via HKDF-SHA256
+- Changing protection level does not retroactively modify existing files
+- `hash` mode detects accidental corruption; `hmac` mode detects tampering; `encrypt` mode provides confidentiality
+
+#### FILE_SECRET
+
+Master secret used for HMAC and encryption operations.
+
+**Default:** None
+
+**Example:**
+
+```bash
+FILE_SECRET=my-strong-secret-key
+```
+
+**Notes:**
+
+- Required when `FILE_PROTECTION_LEVEL` is `hmac` or `encrypt`
+- Treated as a secret (stored via `SecretStr`, supports `/run/secrets` directory)
+- Recommended minimum length: 64 bytes (the HMAC-SHA-512 key size). Generate with: `openssl rand -hex 64`
+- Purpose-specific keys are derived internally via HKDF, so any sufficiently long secret works for both HMAC and encryption
+- Keep this value secure - do not commit to version control
+- Changing the secret will make previously protected files unreadable
+
 ---
 
 ### Vector Database
@@ -987,6 +1036,8 @@ env:
 | `PARSED_JSON_STORE_DIR` | str | No | `json` | JSON subdir |
 | `CHUNKS_STORE_DIR` | str | No | `chunks` | Chunks subdir |
 | `EMBEDDINGS_STORE_DIR` | str | No | `embeddings` | Embeddings subdir |
+| `FILE_PROTECTION_LEVEL` | str | No | `none` | File protection level (none/hash/hmac/encrypt) |
+| `FILE_SECRET` | str | Conditional | - | Master secret for HMAC/encryption (required if protection is hmac or encrypt) |
 | `INGEST_QUEUE_CONCURRENCY` | int | No | `20` | Queue concurrency |
 | `INGEST_WORKER_CONCURRENCY` | int | No | `10` | Worker concurrency |
 | `DOCLING_CONCURRENCY` | int | No | `3` | Docling concurrency |

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "soliplex-ingester"
-version = "0.5.3"
+version = "0.6.0"
 description = "ingestion workflow manager for soliplex"
 authors = [{ name = "Enfold Systems", email = "info@enfoldsystems.net" }]
 readme = "README.md"
@@ -42,6 +42,7 @@ dependencies = [
     "torch>=2.10.0", # added due to missing transitive dependency
     "torchvision>=0.25.0", # added due to missing transitive dependency
     "aiofile>=3.9.0",
+    "cryptography>=44.0.0",
     "sqlalchemy>=2.0.46",
     "pydantic>=2.12.5",
 ]

src/soliplex/ingester/cli.py

@@ -424,5 +424,61 @@ def serve(
         uvicorn.run(app, **uvicorn_kw)
 
 
+async def _apply_protection(level: str):
+    from .lib.config import ProtectionLevel
+    from .lib.dal import apply_file_protection
+
+    try:
+        target = ProtectionLevel(level)
+    except ValueError:
+        print(f"[red]Invalid protection level: {level}[/red]")
+        print(f"Valid options: {', '.join(p.value for p in ProtectionLevel)}")
+        raise typer.Exit(code=1) from None
+
+    settings = get_settings()
+    secret = settings.file_secret.get_secret_value() if settings.file_secret else None
+
+    if target in (ProtectionLevel.HMAC, ProtectionLevel.ENCRYPT) and not secret:
+        print(f"[red]FILE_SECRET is required for protection level '{level}'[/red]")
+        raise typer.Exit(code=1)
+
+    # Secret is also needed to decrypt existing .enc files
+    if target != ProtectionLevel.ENCRYPT and not secret:
+        # Check if any .enc files exist that would need decryption
+        base = Path(settings.file_store_dir)
+        if base.exists():
+            for _p in base.rglob("*.enc"):
+                print("[red]FILE_SECRET is required to decrypt existing encrypted files[/red]")
+                raise typer.Exit(code=1)
+
+    print(f"Applying protection level: [bold]{target.value}[/bold]")
+    stats = await apply_file_protection(target, secret)
+    print(f"Done. processed={stats['processed']} skipped={stats['skipped']} errors={stats['errors']}")
+    if stats["errors"] > 0:
+        raise typer.Exit(code=1)
+
+
+@app.command("protect")
+def protect(
+    level: str = typer.Argument(
+        help="Protection level to apply: none, hash, hmac, encrypt",
+    ),
+):
+    """Apply file protection to all artifacts in the file store.
+
+    Walks all artifact directories, reads each file (decrypting if needed),
+    then writes it with the target protection level and cleans up old
+    protection artifacts.
+
+    Examples:
+        si-cli protect hash       # add SHA-512 hash sidecars
+        si-cli protect hmac       # add HMAC-SHA-512 sidecars
+        si-cli protect encrypt    # encrypt files (requires FILE_SECRET)
+        si-cli protect none       # remove all protection, decrypt files
+    """
+    validate_settings(dump=False)
+    asyncio.run(_apply_protection(level))
+
+
 if __name__ == "__main__":
     app()

src/soliplex/ingester/lib/config.py

@@ -1,3 +1,4 @@
+from enum import StrEnum
 from functools import lru_cache
 from pathlib import Path
 
@@ -7,6 +8,15 @@
 from pydantic_settings import SettingsConfigDict
 
 
+class ProtectionLevel(StrEnum):
+    """Controls integrity/confidentiality protection for file-stored artifacts."""
+
+    NONE = "none"
+    HASH = "hash"
+    HMAC = "hmac"
+    ENCRYPT = "encrypt"
+
+
 class S3Settings(BaseSettings):
     bucket: str = "default"
     endpoint_url: str = "default"
@@ -26,6 +36,8 @@ class Settings(BaseSettings):
     log_level: str = "INFO"
     file_store_target: str = "fs"
     file_store_dir: str = "file_store"
+    file_protection_level: ProtectionLevel = ProtectionLevel.NONE
+    file_secret: SecretStr | None = None
     lancedb_dir: str = "lancedb"
     document_store_dir: str = "raw"
     parsed_markdown_store_dir: str = "markdown"
@@ -80,6 +92,14 @@ def validate_production_auth(self) -> "Settings":
             )
         return self
 
+    @model_validator(mode="after")
+    def validate_file_protection(self) -> "Settings":
+        """Ensure FILE_SECRET is set when protection level requires it."""
+        if self.file_protection_level in (ProtectionLevel.HMAC, ProtectionLevel.ENCRYPT):
+            if not self.file_secret or not self.file_secret.get_secret_value():
+                raise ValueError(f"FILE_SECRET is required when FILE_PROTECTION_LEVEL={self.file_protection_level.value}")
+        return self
+
     worker_checkin_interval: int = 120
     worker_checkin_timeout: int = 600
     worker_task_count: int = 5