Webdav url (#52)

## Summary

update webdav support to improve etag handling

## Changes

- ...

## Test Plan

- [x] Tests pass locally
- [x] Manual testing completed

## Related Issues

Fixes #

Author: bd-mkt

.pre-commit-config.yaml

@@ -13,15 +13,15 @@ repos:
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.14.10
+    rev: v0.15.7
     hooks:
       # Run the linter (reads all config from pyproject.toml).
       - id: ruff
         args: [--fix]
       # Run the formatter (reads line-length from pyproject.toml).
       - id: ruff-format
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.21.2
+    rev: v8.30.1
     hooks:
       - id: gitleaks
   - repo: local
@@ -34,7 +34,7 @@ repos:
         pass_filenames: false
         files: (^pyproject\.toml$|^uv\.lock$)
   - repo: https://github.com/jackdewinter/pymarkdown
-    rev: v0.9.35
+    rev: v0.9.36
     hooks:
       - id: pymarkdown
         args:

README.md

@@ -131,6 +131,9 @@ WEBDAV_URL=https://webdav.example.com
 # WebDAV authentication
 WEBDAV_USERNAME=your-username
 WEBDAV_PASSWORD=your-password
+
+# Disable TLS certificate verification (default: true)
+SSL_VERIFY=true
 ```
 
 All WebDAV credentials can also be provided via command-line options (`--webdav-url`, `--webdav-username`, `--webdav-password`), which override the environment variables.
@@ -155,6 +158,9 @@ AUTH_TRUST_PROXY_HEADERS=false
 
 # Manifest scheduling (requires SCHEDULER_ENABLED=true)
 MANIFEST_DIR=/path/to/manifests
+
+# S3-compatible storage (for urls_file references using s3:// URLs)
+S3_ENDPOINT_URL=https://minio.example.com:9000
 ```
 
 ### Git CLI Mode
@@ -588,7 +594,7 @@ Top-level fields:
 - **name** (required): Component name.
 - **url**: Single URL to fetch.
 - **urls**: List of URLs to fetch.
-- **urls_file**: Path to a file containing URLs (one per line).
+- **urls_file**: Path to a file containing URLs (one per line). Supports local paths, `s3://bucket/key` URLs, and `http(s)://` WebDAV URLs.
 - Exactly one of `url`, `urls`, or `urls_file` must be specified.
 - **extensions**: Override extensions for this component.
 - **metadata**: Additional metadata merged with config-level metadata.
@@ -613,7 +619,7 @@ Top-level fields:
 - **url** (required): WebDAV server URL.
 - **path**: WebDAV directory path to scan recursively.
 - **urls**: List of specific WebDAV file paths to ingest.
-- **urls_file**: Path to a file containing WebDAV URLs (one per line).
+- **urls_file**: Path to a file containing WebDAV URLs (one per line). Supports local paths, `s3://bucket/key` URLs, and `http(s)://` WebDAV URLs (fetched using the same WebDAV credentials).
 - Exactly one of `path`, `urls`, or `urls_file` must be specified.
 - **username**: Override WebDAV username (resolved via Docker secrets or env vars).
 - **password**: Override WebDAV password (resolved via Docker secrets or env vars).
@@ -1165,6 +1171,10 @@ soliplex.agents/
 │   │       ├── webdav.py   # WebDAV API endpoints
 │   │       ├── web.py      # Web API endpoints
 │   │       └── manifest.py # Manifest API endpoints
+│   ├── common/              # Shared utilities
+│   │   ├── urls_file.py     # URL list reader (local, S3, WebDAV)
+│   │   ├── s3.py            # S3 object reader
+│   │   └── config.py        # MIME type detection, config helpers
 │   ├── fs/                 # Filesystem agent
 │   │   ├── app.py          # Core filesystem logic
 │   │   └── cli.py          # Filesystem CLI commands
@@ -1185,7 +1195,7 @@ soliplex.agents/
 │       └── lib/
 │           ├── templates/  # Issue rendering templates
 │           └── utils.py    # Utility functions
-├── example-manifests/      # Example manifest YAML files
+├── example-manifests/      # Example manifests (fs, scm, web, webdav, composite, delete-stale)
 ├── tests/                  # Test suite
 │   └── unit/
 │       ├── test_server_*.py  # Server API tests

src/soliplex/agents/client.py

@@ -250,7 +250,13 @@ async def check_status(file_info: list[dict[str, Any]], source: str, delete_stal
     for x in file_info:
         x["uri"] = x.get("path", x.get("uri"))
 
-    status_dict = {x["uri"]: x["sha256"] for x in file_info}
+    status_dict = {
+        x["uri"]: {
+            "sha256": x.get("sha256") or "",
+            "etag": x.get("_etag") or "",
+        }
+        for x in file_info
+    }
     uri_to_file = {x["uri"]: x for x in file_info}
 
     url = _build_url("/source-status")

src/soliplex/agents/common/urls_file.py

@@ -1,6 +1,7 @@
-"""Shared utility for reading URL list files from local paths or S3."""
+"""Shared utility for reading URL list files from local paths, S3, or WebDAV."""
 
 import logging
+from io import BytesIO
 from pathlib import Path
 
 import aiofiles
@@ -12,6 +13,11 @@
 logger = logging.getLogger(__name__)
 
 
+def is_webdav_url(path: str) -> bool:
+    """Return True if *path* looks like an HTTP(S) URL."""
+    return path.startswith("http://") or path.startswith("https://")
+
+
 def resolve_local_path(
     urls_file: str,
     base_dir: str | None = None,
@@ -42,25 +48,73 @@ def resolve_local_path(
     return urls_file
 
 
+def read_text_from_webdav(
+    url: str,
+    webdav_url: str | None = None,
+    webdav_username: str | None = None,
+    webdav_password: str | None = None,
+) -> str:
+    """Download a text file from a WebDAV server.
+
+    The full file URL is split into a base URL (scheme + host) and a
+    path component.  Authentication credentials fall back to the
+    global settings when not provided explicitly.  Client creation is
+    delegated to :func:`~soliplex.agents.webdav.app.create_webdav_client`
+    so that timeout, header, and TLS settings stay consistent.
+
+    Args:
+        url: Full HTTP(S) URL to the file on the WebDAV server.
+        webdav_url: Optional override for the WebDAV base URL.
+            When *None* the base URL is derived from *url*.
+        webdav_username: Optional WebDAV username.
+        webdav_password: Optional WebDAV password.
+
+    Returns:
+        The file contents decoded as UTF-8 text.
+    """
+    from urllib.parse import urlparse
+
+    from soliplex.agents.webdav.app import create_webdav_client
+
+    parsed = urlparse(url)
+    base_url = webdav_url or f"{parsed.scheme}://{parsed.netloc}"
+    path = parsed.path
+
+    client = create_webdav_client(base_url, webdav_username, webdav_password)
+
+    buffer = BytesIO()
+    client.download_fileobj(path, buffer)
+    return buffer.getvalue().decode("utf-8")
+
+
 async def read_urls_file(
     urls_file: str,
     base_dir: str | None = None,
+    webdav_url: str | None = None,
+    webdav_username: str | None = None,
+    webdav_password: str | None = None,
 ) -> list[str]:
     """Read a URL list file and return non-empty, stripped lines.
 
-    Supports S3 URLs (``s3://bucket/key``) and local filesystem paths.
-    For local paths, relative paths are resolved against *base_dir*
-    when provided (see :func:`resolve_local_path`).
+    Supports S3 URLs (``s3://bucket/key``), WebDAV URLs
+    (``http(s)://...``), and local filesystem paths.  For local paths,
+    relative paths are resolved against *base_dir* when provided (see
+    :func:`resolve_local_path`).
 
     Args:
-        urls_file: Path or S3 URL to the URL list file.
+        urls_file: Path, S3 URL, or WebDAV URL to the URL list file.
         base_dir: Optional directory for resolving relative local paths.
+        webdav_url: Optional WebDAV base URL override (for WebDAV URLs).
+        webdav_username: Optional WebDAV username (for WebDAV URLs).
+        webdav_password: Optional WebDAV password (for WebDAV URLs).
 
     Returns:
         List of non-empty, whitespace-stripped lines.
     """
     if is_s3_url(urls_file):
         content = await read_text_from_s3(urls_file, settings.s3_endpoint_url)
+    elif is_webdav_url(urls_file):
+        content = read_text_from_webdav(urls_file, webdav_url, webdav_username, webdav_password)
     else:
         resolved = resolve_local_path(urls_file, base_dir)
         async with aiofiles.open(resolved) as f:

src/soliplex/agents/manifest/runner.py

@@ -323,7 +323,7 @@ def collect_inventory_uris(result: dict[str, Any]) -> list[dict[str, str]]:
     items: list[dict[str, str]] = []
     for entry in result.get("inventory", []):
         uri = entry.get("uri") or entry.get("path")
-        sha256 = entry.get("sha256", "")
+        sha256 = entry.get("sha256") or ""
         if uri:
             items.append({"uri": uri, "sha256": sha256})
     return items