Skip to content

Commit

Permalink
output falsifying examples
Browse files Browse the repository at this point in the history
  • Loading branch information
bbyalcinkaya committed Jan 6, 2025
1 parent 8c92525 commit 7a2dac8
Showing 1 changed file with 21 additions and 5 deletions.
26 changes: 21 additions & 5 deletions src/komet/kasmer.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,23 +321,26 @@ def deploy_and_run(
print('Selected a single test function:')
print()

failed = []
failed: list[FuzzError] = []
with FuzzProgress(test_bindings, max_examples) as progress:
for task in progress.fuzz_tasks:
try:
task.start()
self.run_test(conf, subst, task.binding, max_examples, task)
task.end()
except AssertionError:
failed.append(task.binding.name)
except FuzzError as e:
failed.append(e)

if not failed:
return

console = Console(stderr=True)

console.print(f'[bold red]{len(failed)}[/bold red] test/s failed:')
for test_name in failed:
console.print(f' {test_name}')

for err in failed:
pretty_args = ', '.join(self.definition.krun.pretty_print(a) for a in err.falsifying_example)
console.print(f' {err.test_name} ({pretty_args})')

raise KSorobanError(failed)

Expand Down Expand Up @@ -469,3 +472,16 @@ def handle_failure(self, args: Mapping[EVar, Pattern]) -> None:
if not self.failed:
self.failed = True
self.task.fail()

sorted_keys = sorted(args.keys(), key=lambda k: k.name)
falsifying = tuple(self.definition.krun.kore_to_kast(args[k]) for k in sorted_keys)
raise FuzzError(self.task.binding.name, falsifying)


class FuzzError(Exception):
test_name: str
falsifying_example: tuple[KInner, ...]

def __init__(self, test_name: str, falsifying_example: tuple[KInner, ...]):
self.test_name = test_name
self.falsifying_example = falsifying_example

0 comments on commit 7a2dac8

Please sign in to comment.