Testdata statistics

engine.go

@@ -263,6 +263,9 @@ func findBug(tb tb, checks int, seed uint64, prop func(*T)) (uint64, int, int, *
 			return seed, valid, invalid, err
 		}
 	}
+	// Finally, print the stats
+	// TODO; Does not print the stats, but seems to count correctly?
+	printStats(t)
 
 	return 0, valid, invalid, nil
 }
@@ -275,7 +278,6 @@ func checkOnce(t *T, prop func(*T)) (err *testError) {
 
 	prop(t)
 	t.failOnError()
-
 	return nil
 }
 
@@ -409,6 +411,8 @@ type T struct {
 	refDraws []value
 	mu       sync.RWMutex
 	failed   stopTest
+	evChan   chan event
+	evDone   chan done
 }
 
 func newT(tb tb, s bitStream, tbLog bool, rawLog *log.Logger, refDraws ...value) *T {

event.go

@@ -0,0 +1,110 @@
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+package rapid
+
+import (
+	"log"
+	"sort"
+)
+
+// counter maps a (stringified) event to a frequency counter
+type counter map[string]int
+
+// stats maps labels to counters
+type stats map[string]counter
+
+type event struct {
+	label string
+	value string
+}
+type done struct {
+	result chan stats
+}
+
+type counterPair struct {
+	frequency int
+	event     string
+}
+
+// Event records an event for test `t` and
+// stores the event for calculating statistics.
+//
+// Recording events and printing a their statistic is a tool for
+// analysing test data generations. It helps to understand if
+// your customer generators produce value in the expected range.
+//
+// Each event has a label and an event value. To see the statistics,
+// run the tests with `go test -v`.
+//
+func Event(t *T, label string, value string) {
+	t.Helper()
+	if t.evChan == nil {
+		log.Printf("Creating the channels for test %s", t.Name())
+		t.evChan = make(chan event)
+		t.evDone = make(chan done)
+		go eventRecorder(t.evChan, t.evDone)
+	}
+	ev := event{value: value, label: label}
+	// log.Printf("Send the event %+v", ev)
+	t.evChan <- ev
+}
+
+// eventRecorder is a goroutine that stores event for a test execution.
+func eventRecorder(incomingEvent <-chan event, done <-chan done) {
+	all_stats := make(stats)
+	for {
+		select {
+		case ev := <-incomingEvent:
+			c, found := all_stats[ev.label]
+			if !found {
+				c = make(counter)
+				all_stats[ev.label] = c
+			}
+			c[ev.value]++
+		case d := <-done:
+			log.Printf("event Recorder: Done. Send the stats\n")
+			d.result <- all_stats
+			log.Printf("event Recorder: Done. Will return now\n")
+			return
+		}
+	}
+	// log.Printf("event Recorder: This shall never happen\n")
+
+}
+
+// printStats logs a table of events and their relative frequency.
+func printStats(t *T) {
+	// log.Printf("What about printing the stats for t = %+v", t)
+	if t.evChan == nil || t.evDone == nil {
+		return
+	}
+	log.Printf("Now we can print the stats")
+	d := done{result: make(chan stats)}
+	t.evDone <- d
+	stats := <-d.result
+	log.Printf("stats received")
+	log.Printf("Statistics for %s\n", t.Name())
+	for label := range stats {
+		log.Printf("Events with label %s", label)
+		s := stats[label]
+		events := make([]counterPair, 0)
+		sum := 0
+		count := 0
+		for ev := range s {
+			sum += s[ev]
+			count++
+			events = append(events, counterPair{event: ev, frequency: s[ev]})
+		}
+		log.Printf("Total of %d different events\n", count)
+		// we sort twice to sort same frequency alphabetically
+		sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event })
+		sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency })
+		for _, ev := range events {
+			log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0)
+		}
+	}
+	close(t.evChan)
+	close(t.evDone)
+}

event_test.go

@@ -0,0 +1,47 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package rapid
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+func TestEventEmitter(t *testing.T) {
+	t.Parallel()
+	Check(t, func(te *T) {
+		// te.rawLog.SetOutput(te.rawLog.Output())
+		Event(te, "var", "x")
+		Event(te, "var", "y")
+
+		// checkMatch(te, fmt.Sprintf("Statistics.*%s", te.Name()), te.output[0])
+		// checkMatch(te, "of 2 ", te.output[1])
+		// checkMatch(te, "x: 1 \\(50.0+ %", te.output[3])
+		// checkMatch(te, "y: 1 \\(50.0+ %", te.output[4])
+
+	})
+}
+
+func checkMatch(t *T, pattern, str string) {
+	matched, err := regexp.MatchString(pattern, str)
+	if err != nil {
+		t.Fatalf("Regex compile failed")
+	}
+	if !matched {
+		t.Fatalf("Pattern <%s> does not match in <%s>", pattern, str)
+	}
+}
+
+func TestTrivialPropertyWithEvents(t *testing.T) {
+	t.Parallel()
+	Check(t, func(te *T) {
+		x := Uint8().Draw(te, "x").(uint8)
+		Event(te, "x", fmt.Sprintf("%d", x))
+		if x > 255 {
+			t.Fatalf("x should fit into a byte")
+		}
+	})
+}

example_event_test.go

@@ -0,0 +1,30 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package rapid_test
+
+import (
+	"fmt"
+	"testing"
+
+	"pgregory.net/rapid"
+)
+
+func ExampleEvent(t *testing.T) {
+	rapid.Check(t, func(t *rapid.T) {
+		// For any integers x, y ...
+		x := rapid.Int().Draw(t, "x").(int)
+		y := rapid.Int().Draw(t, "y").(int)
+		// ... report them ...
+		rapid.Event(t, "x", fmt.Sprintf("%d", x))
+		rapid.Event(t, "y", fmt.Sprintf("%d", y))
+
+		// ... the property holds
+		if x+y != y+x {
+			t.Fatalf("associativty of + does not hold")
+		}
+		// statistics are printed after the property (if called with go test -v)
+	})
+	// Output:
+}