Remove W&B/vLLM/SGLang; fix canary --provider, Qdrant text, Ray nodes, networking

Author: theoddden

.github/workflows/ml-drift.yml

@@ -62,13 +62,13 @@ jobs:
           run_drift ollama
           stop_and_remove ollama
 
-          # Weaviate
+          # Weaviate (wait until it reports itself ready, not just live)
           docker run -d --rm --name weaviate -p 8080:8080 \
             -e AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true \
             -e AUTHORIZATION_ENABLED=false \
             cr.weaviate.io/semitechnologies/weaviate:latest
-          for i in $(seq 1 60); do
-            curl -fs http://localhost:8080/v1/.well-known/live > /dev/null && break
+          for i in $(seq 1 120); do
+            curl -fs http://localhost:8080/v1/.well-known/ready > /dev/null && break
             sleep 1
           done
           run_drift weaviate
@@ -87,56 +87,48 @@ jobs:
           # Letta
           docker run -d --rm --name letta -p 8283:8283 \
             -e OPENAI_API_KEY=dummy letta/letta:latest
-          for i in $(seq 1 60); do
+          for i in $(seq 1 120); do
             curl -fs http://localhost:8283/v1/health/ > /dev/null && break
             sleep 1
           done
+          if ! curl -fs http://localhost:8283/v1/health/ > /dev/null 2>&1; then
+            echo "Letta failed to become ready; last logs:"
+            docker logs letta
+          fi
           run_drift letta
           stop_and_remove letta
 
-          # Langfuse (requires Postgres)
-          docker run -d --rm --name postgres -p 5432:5432 \
+          # Langfuse (Postgres on a Docker network so langfuse can resolve postgres by name)
+          docker network create ml-drift-net >/dev/null 2>&1 || true
+          docker run -d --rm --name postgres --network ml-drift-net \
             -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres \
             -e POSTGRES_DB=langfuse postgres:15
           for i in $(seq 1 60); do
             docker exec postgres pg_isready -U postgres > /dev/null 2>&1 && break
             sleep 1
           done
-          docker run -d --rm --name langfuse -p 3000:3000 \
-            -e DATABASE_URL=postgresql://postgres:postgres@localhost:5432/langfuse \
+          docker run -d --rm --name langfuse --network ml-drift-net -p 3000:3000 \
+            -e DATABASE_URL=postgresql://postgres:postgres@postgres:5432/langfuse \
             -e NEXTAUTH_URL=http://localhost:3000 \
             -e NEXTAUTH_SECRET=secret \
             -e SALT=salt \
             -e ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 \
             -e TELEMETRY_ENABLED=false \
             -e LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=false \
             langfuse/langfuse:latest
-          for i in $(seq 1 60); do
+          for i in $(seq 1 120); do
             curl -fs http://localhost:3000/api/public/health > /dev/null && break
             sleep 1
           done
+          if ! curl -fs http://localhost:3000/api/public/health > /dev/null 2>&1; then
+            echo "Langfuse failed to become ready; last logs:"
+            docker logs langfuse
+            docker logs postgres
+          fi
           run_drift langfuse
           stop_and_remove langfuse
           stop_and_remove postgres
-
-          # W&B local requires a license and ~4GB RAM
-          if [ -n "${{ secrets.WANDB_LICENSE }}" ]; then
-            docker run -d --rm --name wandb -p 8080:8080 \
-              -e LICENSE=${{ secrets.WANDB_LICENSE }} \
-              -e HOST=http://localhost:8080 \
-              wandb/local
-            for i in $(seq 1 90); do
-              curl -fs http://localhost:8080/healthz > /dev/null && break
-              sleep 2
-            done
-            run_drift wandb
-            stop_and_remove wandb
-          else
-            echo "WANDB_LICENSE secret not set; skipping W&B local (requires license and 4GB+ RAM)"
-          fi
-
-          # vLLM and SGLang require a GPU; GitHub-hosted runners do not have one.
-          echo "vLLM and SGLang require a GPU runner; skipping on CPU-only runner"
+          docker network rm ml-drift-net >/dev/null 2>&1 || true
 
           if [ "$failed" -ne 0 ]; then
             exit 1

terradev_cli/commands/canary.py

@@ -287,7 +287,6 @@ def canary_tail(file, limit):
     "aws": ["TERRADEV_AWS_ACCESS_KEY_ID"],
     "gcp": ["TERRADEV_GCP_CREDENTIALS"],
     "inferx": ["TERRADEV_INFERX_API_KEY"],
-    "wandb": ["TERRADEV_WANDB_API_KEY"],
     "langfuse": ["TERRADEV_LANGFUSE_PUBLIC_KEY"],
     "letta": ["TERRADEV_LETTA_API_KEY"],
     "weaviate": ["TERRADEV_WEAVIATE_API_KEY"],
@@ -420,9 +419,6 @@ def _load_drift_env_extras(alias: str, api_key: str) -> Optional[Dict[str, Any]]
         extras["secret_key"] = secret_key
         extras["api_key"] = public_key
 
-    elif alias == "wandb":
-        extras["api_key"] = api_key.strip()
-
     elif alias == "letta":
         extras["api_key"] = api_key.strip()
         extras["bearer_token"] = api_key.strip()
@@ -618,7 +614,9 @@ def canary_drift(ctx, drift_all, provider, contracts_dir, drift_format, drift_ti
         providers.append(contract.get("provider") or p.stem)
     credentials = {} if no_credentials else _load_drift_credentials(providers)
     monitor = DriftMonitor(str(contracts_path), credentials, timeout=drift_timeout)
-    monitor.run_all()
+    monitor.results = []
+    for p in contract_files:
+        monitor.results.append(monitor.check_provider(p))
     summary = monitor.summary()
 
     out = get_output(ctx)
@@ -740,7 +738,9 @@ def canary_ml_drift(ctx, ml_all, provider, drift_format, drift_timeout, base_url
         timeout=drift_timeout,
         base_url_overrides=overrides,
     )
-    monitor.run_all()
+    monitor.results = []
+    for p in contract_files:
+        monitor.results.append(monitor.check_provider(p))
     summary = monitor.summary()
 
     out = get_output(ctx)

terradev_cli/drift_monitor/ml_service_contracts/qdrant.yaml

@@ -9,14 +9,14 @@ endpoints:
   enabled: true
   expected_status: 200
   content_type: text/plain
-  expected_text: "ok"
+  expected_text: "all shards are ready"
 - name: livez
   method: GET
   path: "livez"
   enabled: true
   expected_status: 200
   content_type: text/plain
-  expected_text: "ok"
+  expected_text: "healthz check passed"
 - name: collections
   method: GET
   path: "collections"

terradev_cli/drift_monitor/ml_service_contracts/ray.yaml

@@ -14,10 +14,11 @@ endpoints:
   - clusterStatus
 - name: nodes
   method: GET
-  path: "api/nodes"
+  path: "api/v0/nodes"
   enabled: true
   expected_status: 200
   expected_response_fields:
+  - result
   - data
   optional_response_fields:
   - raylet

terradev_cli/drift_monitor/ml_service_contracts/sglang.yaml

@@ -9,12 +9,6 @@
 class TestMlDriftCredentials:
     """Verify TERRADEV_* tokens for new ML services are loaded correctly."""
 
-    def test_wandb_api_key_loaded(self, monkeypatch):
-        monkeypatch.setenv("TERRADEV_WANDB_API_KEY", "wandb-key")
-        creds = _load_drift_credentials(["wandb"])
-        assert "wandb" in creds
-        assert creds["wandb"]["api_key"] == "wandb-key"
-
     def test_langfuse_public_and_secret_keys_loaded(self, monkeypatch):
         monkeypatch.setenv("TERRADEV_LANGFUSE_PUBLIC_KEY", "pk-lf-abc")
         monkeypatch.setenv("TERRADEV_LANGFUSE_SECRET_KEY", "sk-lf-xyz")
@@ -39,6 +33,6 @@ def test_weaviate_api_key_loaded(self, monkeypatch):
         assert creds["weaviate"]["bearer_token"] == "weaviate-key"
 
     def test_missing_tokens_return_empty(self, monkeypatch):
-        monkeypatch.delenv("TERRADEV_WANDB_API_KEY", raising=False)
-        creds = _load_drift_credentials(["wandb"])
-        assert "wandb" not in creds
+        monkeypatch.delenv("TERRADEV_WEAVIATE_API_KEY", raising=False)
+        creds = _load_drift_credentials(["weaviate"])
+        assert "weaviate" not in creds