perf(tok): pre-size terms slice in uniqueTerms

Pre-size the terms slice to len(tokens) instead of growing from nil. Also
un-skips BenchmarkTermTokenizer with a real workload.

BenchmarkTermTokenizer: allocs -24% geomean, B/op -14% geomean.

Author: shaunpatterson

tok/bleve.go

@@ -61,7 +61,7 @@ func setupBleve() {
 
 // uniqueTerms takes a token stream and returns a string slice of unique terms.
 func uniqueTerms(tokens analysis.TokenStream) []string {
-	var terms []string
+	terms := make([]string, 0, len(tokens))
 	for i := range tokens {
 		terms = append(terms, string(tokens[i].Term))
 	}

tok/tok_test.go

@@ -6,6 +6,7 @@
 package tok
 
 import (
+	"fmt"
 	"math"
 	"sort"
 	"strings"
@@ -653,5 +654,20 @@ func TestNGramTokenizerNonStringInput(t *testing.T) {
 }
 
 func BenchmarkTermTokenizer(b *testing.B) {
-	b.Skip() // tmp
+	t := TermTokenizer{}
+	b.ReportAllocs()
+	for _, text := range []string{
+		"the quick brown fox jumps over the lazy dog",
+		"Lorem ipsum dolor sit amet, consectetur adipiscing elit",
+		"a b c d e f g h i j k l m n o p q r s t u v w x y z",
+	} {
+		b.Run(fmt.Sprintf("len=%d", len(text)), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				_, err := t.Tokens(text)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
 }