Skip to content

Commit 4262574

Browse files
DannyWall92claude
andcommitted
Run ruff format on inference-monitor and training-monitor
The lint fixes introduced formatting inconsistencies that ruff format --check catches in CI. Run the formatter to bring all files in line. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 53b4eee commit 4262574

20 files changed

Lines changed: 115 additions & 181 deletions

inference-monitor/src/sentinel_inference/analyzers/kl_divergence.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -162,9 +162,7 @@ def submit(
162162

163163
if divergence > self._config.threshold_nats:
164164
severity = (
165-
"critical"
166-
if divergence > self._config.threshold_nats * 10
167-
else "warning"
165+
"critical" if divergence > self._config.threshold_nats * 10 else "warning"
168166
)
169167
anomalies.append(
170168
AnomalyEvent(

inference-monitor/src/sentinel_inference/analyzers/logit_analyzer.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,10 @@ class EWMATracker:
6767
)
6868

6969
def __init__(
70-
self, lambda_: float = 0.1, control_limit: float = 3.5, burn_in: int = 1000,
70+
self,
71+
lambda_: float = 0.1,
72+
control_limit: float = 3.5,
73+
burn_in: int = 1000,
7174
) -> None:
7275
self._lambda = lambda_
7376
self._L = control_limit
@@ -129,7 +132,7 @@ def update(self, value: float) -> bool:
129132

130133
# Welford-style running mean and variance for target & sigma
131134
self._target = self._sum / self._count
132-
variance = (self._sum_sq / self._count) - (self._target ** 2)
135+
variance = (self._sum_sq / self._count) - (self._target**2)
133136
self._sigma = np.sqrt(max(variance, 1e-15))
134137

135138
# EWMA update

inference-monitor/src/sentinel_inference/analyzers/statistical_tests.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,7 @@ def baseline_size(self) -> int:
4444
def _rebuild_baseline(self) -> None:
4545
"""Concatenate baseline samples into a single array for testing."""
4646
if self._baseline:
47-
self._baseline_flat = np.concatenate(
48-
[s.ravel() for s in self._baseline]
49-
)
47+
self._baseline_flat = np.concatenate([s.ravel() for s in self._baseline])
5048
else:
5149
self._baseline_flat = None
5250
self._needs_rebuild = False

inference-monitor/src/sentinel_inference/grpc_client.py

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,14 @@
3737

3838
# Anomaly type mapping from analyzer names to proto enum values.
3939
_ANOMALY_TYPE_MAP: dict[str, int] = {
40-
"logit_analyzer": 1, # ANOMALY_TYPE_LOGIT_DRIFT
41-
"entropy_analyzer": 2, # ANOMALY_TYPE_ENTROPY_ANOMALY
42-
"kl_divergence": 3, # ANOMALY_TYPE_KL_DIVERGENCE
43-
"gradient_monitor": 4, # ANOMALY_TYPE_GRADIENT_NORM_SPIKE
44-
"loss_monitor": 5, # ANOMALY_TYPE_LOSS_SPIKE
45-
"ddp_divergence": 6, # ANOMALY_TYPE_CROSS_RANK_DIVERGENCE
40+
"logit_analyzer": 1, # ANOMALY_TYPE_LOGIT_DRIFT
41+
"entropy_analyzer": 2, # ANOMALY_TYPE_ENTROPY_ANOMALY
42+
"kl_divergence": 3, # ANOMALY_TYPE_KL_DIVERGENCE
43+
"gradient_monitor": 4, # ANOMALY_TYPE_GRADIENT_NORM_SPIKE
44+
"loss_monitor": 5, # ANOMALY_TYPE_LOSS_SPIKE
45+
"ddp_divergence": 6, # ANOMALY_TYPE_CROSS_RANK_DIVERGENCE
4646
"checkpoint_validator": 7, # ANOMALY_TYPE_CHECKPOINT_DIVERGENCE
47-
"invariant_checker": 8, # ANOMALY_TYPE_INVARIANT_VIOLATION
47+
"invariant_checker": 8, # ANOMALY_TYPE_INVARIANT_VIOLATION
4848
}
4949

5050
# Severity mapping from string labels to proto enum values.
@@ -174,8 +174,11 @@ async def start(self) -> None:
174174

175175
# Try to import the generated protobuf stubs.
176176
self._stub = self._create_stub()
177-
logger.info("grpc_client_started", endpoint=self._config.endpoint,
178-
proto_stubs=self._proto_available)
177+
logger.info(
178+
"grpc_client_started",
179+
endpoint=self._config.endpoint,
180+
proto_stubs=self._proto_available,
181+
)
179182
except ImportError:
180183
logger.warning("grpc_not_available", msg="Running without gRPC reporting.")
181184
self._channel = None
@@ -187,13 +190,15 @@ def _create_stub(self) -> Any:
187190
"""Create the AnomalyService stub, trying generated stubs first."""
188191
try:
189192
from sentinel.v1 import anomaly_pb2_grpc # type: ignore[import-untyped]
193+
190194
stub = anomaly_pb2_grpc.AnomalyServiceStub(self._channel)
191195
self._proto_available = True
192196
logger.debug("using_generated_proto_stubs")
193197
return stub
194198
except ImportError:
195-
logger.debug("generated_proto_stubs_not_found",
196-
msg="Falling back to manual serialization")
199+
logger.debug(
200+
"generated_proto_stubs_not_found", msg="Falling back to manual serialization"
201+
)
197202
self._proto_available = False
198203
return None
199204

@@ -217,8 +222,7 @@ async def stop(self) -> None:
217222
self._stream = None
218223
if self._channel is not None:
219224
await self._channel.close()
220-
logger.info("grpc_client_stopped",
221-
pending_acks=len(self._pending_acks))
225+
logger.info("grpc_client_stopped", pending_acks=len(self._pending_acks))
222226

223227
# ------------------------------------------------------------------
224228
# Event submission
@@ -283,9 +287,7 @@ def _event_to_proto_dict(self, event: AnomalyEvent) -> dict[str, Any]:
283287
else:
284288
severity_val = _SEVERITY_MAP.get(str(severity_str).lower(), 2)
285289

286-
anomaly_type_val = _ANOMALY_TYPE_MAP.get(
287-
getattr(event, "analyzer", ""), 0
288-
)
290+
anomaly_type_val = _ANOMALY_TYPE_MAP.get(getattr(event, "analyzer", ""), 0)
289291

290292
# Generate a deterministic event ID from content hash.
291293
content = f"{event.analyzer}:{event.stat_name}:{event.sample_count}:{event.observed_value}"
@@ -307,9 +309,7 @@ def _event_to_proto_dict(self, event: AnomalyEvent) -> dict[str, Any]:
307309
f"{event.analyzer}/{event.stat_name}: observed={event.observed_value:.6f}, "
308310
f"ewma={event.ewma_value:.6f}, ucl={event.ucl}, lcl={event.lcl}"
309311
),
310-
"tensor_fingerprint": hashlib.sha256(
311-
f"{event.observed_value}".encode()
312-
).digest()[:16],
312+
"tensor_fingerprint": hashlib.sha256(f"{event.observed_value}".encode()).digest()[:16],
313313
"timestamp": time.time(),
314314
"metadata": event.details if isinstance(event.details, dict) else {},
315315
"step_number": event.sample_count,
@@ -428,9 +428,7 @@ async def _do_send_proto(self, report: AnomalyReport) -> None:
428428
if self._stream is None:
429429
self._stream = self._stub.StreamAnomalyEvents()
430430
# Start background ack reader.
431-
self._ack_task = asyncio.create_task(
432-
self._read_acks_proto()
433-
)
431+
self._ack_task = asyncio.create_task(self._read_acks_proto())
434432

435433
self._pending_acks[report.sequence_number] = time.time()
436434
await self._stream.write(batch)
@@ -470,13 +468,15 @@ async def _do_send_manual(self, report: AnomalyReport) -> None:
470468
"""
471469
import json
472470

473-
474-
payload = json.dumps({
475-
"source_hostname": report.source_hostname,
476-
"sequence_number": report.sequence_number,
477-
"batch_timestamp": {"seconds": int(report.timestamp)},
478-
"events": report.events,
479-
}, default=str).encode("utf-8")
471+
payload = json.dumps(
472+
{
473+
"source_hostname": report.source_hostname,
474+
"sequence_number": report.sequence_number,
475+
"batch_timestamp": {"seconds": int(report.timestamp)},
476+
"events": report.events,
477+
},
478+
default=str,
479+
).encode("utf-8")
480480

481481
call = self._channel.unary_unary(
482482
"/sentinel.v1.AnomalyService/StreamAnomalyEvents",

inference-monitor/src/sentinel_inference/interceptors/triton_interceptor.py

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,16 @@ async def capture_output(self) -> TensorCapture | None:
125125
for s in shape:
126126
num_elements *= s
127127

128-
tensor = np.frombuffer(
129-
buf,
130-
dtype=np.float32,
131-
count=num_elements,
132-
offset=offset,
133-
).reshape(shape).copy()
128+
tensor = (
129+
np.frombuffer(
130+
buf,
131+
dtype=np.float32,
132+
count=num_elements,
133+
offset=offset,
134+
)
135+
.reshape(shape)
136+
.copy()
137+
)
134138

135139
# Clear ready flag
136140
struct.pack_into("<I", buf, 4, flags & ~1)

inference-monitor/src/sentinel_inference/monitor.py

Lines changed: 13 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -246,16 +246,15 @@ async def _analyzer_worker(self, worker_id: int) -> None:
246246
all_anomalies: list[AnomalyEvent] = []
247247

248248
try:
249-
all_anomalies.extend(
250-
await self._run_analysis(capture, labels)
251-
)
249+
all_anomalies.extend(await self._run_analysis(capture, labels))
252250
except Exception:
253251
logger.exception("analyzer_worker_error", worker_id=worker_id)
254252

255253
if all_anomalies:
256254
for event in all_anomalies:
257255
ANOMALIES_TOTAL.labels(
258-
**labels, type=event.analyzer,
256+
**labels,
257+
type=event.analyzer,
259258
).inc()
260259
await self._grpc_client.submit(all_anomalies)
261260

@@ -274,22 +273,14 @@ async def _run_analysis(
274273

275274
# Logit analyzer
276275
t0 = time.monotonic()
277-
result = await loop.run_in_executor(
278-
None, self._logit_analyzer.analyze, capture.tensor
279-
)
280-
ANALYSIS_DURATION.labels(**labels, analyzer="logit").observe(
281-
time.monotonic() - t0
282-
)
276+
result = await loop.run_in_executor(None, self._logit_analyzer.analyze, capture.tensor)
277+
ANALYSIS_DURATION.labels(**labels, analyzer="logit").observe(time.monotonic() - t0)
283278
anomalies.extend(result)
284279

285280
# Entropy analyzer
286281
t0 = time.monotonic()
287-
result = await loop.run_in_executor(
288-
None, self._entropy_analyzer.analyze, capture.tensor
289-
)
290-
ANALYSIS_DURATION.labels(**labels, analyzer="entropy").observe(
291-
time.monotonic() - t0
292-
)
282+
result = await loop.run_in_executor(None, self._entropy_analyzer.analyze, capture.tensor)
283+
ANALYSIS_DURATION.labels(**labels, analyzer="entropy").observe(time.monotonic() - t0)
293284
anomalies.extend(result)
294285

295286
# KL divergence (cross-replica)
@@ -302,35 +293,25 @@ async def _run_analysis(
302293
capture.request_id,
303294
None,
304295
)
305-
ANALYSIS_DURATION.labels(**labels, analyzer="kl_divergence").observe(
306-
time.monotonic() - t0
307-
)
296+
ANALYSIS_DURATION.labels(**labels, analyzer="kl_divergence").observe(time.monotonic() - t0)
308297
anomalies.extend(result)
309298

310299
# Spectral analyzer
311300
t0 = time.monotonic()
312-
result = await loop.run_in_executor(
313-
None, self._spectral_analyzer.analyze, capture.tensor
314-
)
315-
ANALYSIS_DURATION.labels(**labels, analyzer="spectral").observe(
316-
time.monotonic() - t0
317-
)
301+
result = await loop.run_in_executor(None, self._spectral_analyzer.analyze, capture.tensor)
302+
ANALYSIS_DURATION.labels(**labels, analyzer="spectral").observe(time.monotonic() - t0)
318303
anomalies.extend(result)
319304

320305
# Statistical tests
321306
t0 = time.monotonic()
322-
result = await loop.run_in_executor(
323-
None, self._stat_tests.analyze, capture.tensor
324-
)
307+
result = await loop.run_in_executor(None, self._stat_tests.analyze, capture.tensor)
325308
ANALYSIS_DURATION.labels(**labels, analyzer="statistical_tests").observe(
326309
time.monotonic() - t0
327310
)
328311
anomalies.extend(result)
329312

330313
# Tensor fingerprint (for logging / future use)
331-
_fp = await loop.run_in_executor(
332-
None, self._fingerprinter.compute, capture.tensor
333-
)
314+
_fp = await loop.run_in_executor(None, self._fingerprinter.compute, capture.tensor)
334315

335316
# Update token sketches if tensor looks like token IDs
336317
if capture.tensor.ndim >= 1:
@@ -408,6 +389,7 @@ def main() -> None:
408389
if sys.platform != "win32":
409390
try:
410391
import uvloop
392+
411393
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
412394
logger.info("using_uvloop")
413395
except ImportError:

inference-monitor/tests/unit/test_kl_divergence.py

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,7 @@ def test_identical_distributions(self) -> None:
6565
def test_symmetric(self) -> None:
6666
p = np.array([0.9, 0.1])
6767
q = np.array([0.1, 0.9])
68-
assert abs(
69-
jensen_shannon_divergence(p, q) - jensen_shannon_divergence(q, p)
70-
) < 1e-10
68+
assert abs(jensen_shannon_divergence(p, q) - jensen_shannon_divergence(q, p)) < 1e-10
7169

7270
def test_bounded(self) -> None:
7371
"""JSD should be in [0, ln(2)]."""
@@ -96,15 +94,11 @@ def test_no_anomaly_same_output(self) -> None:
9694
input_hash = "test_hash_1"
9795

9896
# Submit from replica A
99-
events_a = detector.submit(
100-
logits, input_hash, "req1", replica_id="replica_a"
101-
)
97+
events_a = detector.submit(logits, input_hash, "req1", replica_id="replica_a")
10298
assert len(events_a) == 0
10399

104100
# Submit same logits from replica B
105-
events_b = detector.submit(
106-
logits, input_hash, "req2", replica_id="replica_b"
107-
)
101+
events_b = detector.submit(logits, input_hash, "req2", replica_id="replica_b")
108102
assert len(events_b) == 0
109103

110104
def test_detects_divergent_outputs(self) -> None:
@@ -118,9 +112,7 @@ def test_detects_divergent_outputs(self) -> None:
118112
input_hash = "test_hash_2"
119113

120114
detector.submit(logits_a, input_hash, "req1", replica_id="replica_a")
121-
events = detector.submit(
122-
logits_b, input_hash, "req2", replica_id="replica_b"
123-
)
115+
events = detector.submit(logits_b, input_hash, "req2", replica_id="replica_b")
124116
assert len(events) > 0
125117
assert events[0].analyzer == "kl_divergence"
126118

@@ -135,9 +127,7 @@ def test_same_replica_ignored(self) -> None:
135127
input_hash = "test_hash_3"
136128

137129
detector.submit(logits_a, input_hash, "req1", replica_id="replica_a")
138-
events = detector.submit(
139-
logits_b, input_hash, "req2", replica_id="replica_a"
140-
)
130+
events = detector.submit(logits_b, input_hash, "req2", replica_id="replica_a")
141131
assert len(events) == 0
142132

143133
def test_different_input_hash_not_matched(self) -> None:

inference-monitor/tests/unit/test_logit_analyzer.py

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,7 @@ class TestLogitAnalyzer:
136136
"""Integration tests for the full LogitAnalyzer."""
137137

138138
def test_no_anomalies_during_burn_in(self) -> None:
139-
config = LogitAnalyzerConfig(
140-
ewma=EWMAConfig(**{"lambda": 0.1, "L": 3.5, "burn_in": 100})
141-
)
139+
config = LogitAnalyzerConfig(ewma=EWMAConfig(**{"lambda": 0.1, "L": 3.5, "burn_in": 100}))
142140
analyzer = LogitAnalyzer(config)
143141
rng = np.random.default_rng(42)
144142
for _ in range(99):
@@ -148,9 +146,7 @@ def test_no_anomalies_during_burn_in(self) -> None:
148146

149147
def test_detects_corruption(self) -> None:
150148
"""Analyzer should detect a sudden distributional shift."""
151-
config = LogitAnalyzerConfig(
152-
ewma=EWMAConfig(**{"lambda": 0.1, "L": 3.0, "burn_in": 50})
153-
)
149+
config = LogitAnalyzerConfig(ewma=EWMAConfig(**{"lambda": 0.1, "L": 3.0, "burn_in": 50}))
154150
analyzer = LogitAnalyzer(config)
155151
rng = np.random.default_rng(42)
156152

@@ -176,9 +172,7 @@ def test_disabled_returns_empty(self) -> None:
176172
assert analyzer.analyze(logits) == []
177173

178174
def test_anomaly_event_fields(self) -> None:
179-
config = LogitAnalyzerConfig(
180-
ewma=EWMAConfig(**{"lambda": 0.3, "L": 2.0, "burn_in": 20})
181-
)
175+
config = LogitAnalyzerConfig(ewma=EWMAConfig(**{"lambda": 0.3, "L": 2.0, "burn_in": 20}))
182176
analyzer = LogitAnalyzer(config)
183177
rng = np.random.default_rng(0)
184178

@@ -187,9 +181,7 @@ def test_anomaly_event_fields(self) -> None:
187181

188182
# Inject anomaly
189183
for _ in range(50):
190-
anomalies = analyzer.analyze(
191-
rng.normal(20.0, 1.0, size=500).astype(np.float32)
192-
)
184+
anomalies = analyzer.analyze(rng.normal(20.0, 1.0, size=500).astype(np.float32))
193185
if anomalies:
194186
event = anomalies[0]
195187
assert event.analyzer == "logit_analyzer"

training-monitor/src/sentinel_training/common/anomaly_detector.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,7 @@ def is_sdc_suspected(self) -> bool:
220220
"""
221221
return self.composite_score() > self._composite_threshold
222222

223-
def recent_anomalies(
224-
self, anomaly_type: AnomalyType | None = None
225-
) -> list[AnomalyScore]:
223+
def recent_anomalies(self, anomaly_type: AnomalyType | None = None) -> list[AnomalyScore]:
226224
"""Return recent anomalies, optionally filtered by type.
227225
228226
Args:

0 commit comments

Comments
 (0)