feat: add vale test

Refs #1122

Signed-off-by: Joseph Kato <joseph@jdkato.io>

Author: jdkato

cmd/vale/command.go

@@ -37,6 +37,7 @@ var commandInfo = map[string]string{
 	"host-install":   "Install the Vale native messaging host for the given browser.",
 	"host-uninstall": "Uninstall the Vale native messaging host for the given browser.",
 	"fix":            "Attempt to automatically fix the given alert.",
+	"test":           "Run the test cases kept beside a configuration's rules.",
 }
 
 // Actions are the available CLI commands.
@@ -46,6 +47,7 @@ var Actions = map[string]func(args []string, flags *core.CLIFlags) error{
 	"ls-dirs":    printDirs,
 	"ls-vars":    printVars,
 	"sync":       sync,
+	"test":       runTests,
 
 	// private
 	"host-install":   installNativeHost,

cmd/vale/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -104,10 +105,18 @@ func main() {
 	if argc > 0 {
 		cmd, exists := Actions[args[0]]
 		if exists {
-			if err := cmd(args[1:], &Flags); err != nil {
+			err := cmd(args[1:], &Flags)
+			stopProfiling()
+
+			// Failing test cases mean Vale worked and the configuration did
+			// not, which is the same distinction `vale file.md` draws between
+			// exiting 1 and exiting 2. The command has already reported them.
+			if errors.Is(err, errTestFailed) {
+				os.Exit(1)
+			} else if err != nil {
 				handleError(err)
 			}
-			stopProfiling()
+
 			os.Exit(0)
 		}
 	}

cmd/vale/test.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pterm/pterm"
+
+	"github.com/errata-ai/vale/v3/internal/core"
+	"github.com/errata-ai/vale/v3/internal/testsuite"
+)
+
+// errTestFailed reports cases that ran and did not pass.
+//
+// It is not an error in the sense the others are -- Vale worked, the
+// configuration did not -- so main exits 1 with it rather than printing it as
+// a failure of Vale's own.
+var errTestFailed = errors.New("test cases failed")
+
+// testReport is the JSON form of a run.
+type testReport struct {
+	Passed  int          `json:"passed"`
+	Failed  int          `json:"failed"`
+	Results []testResult `json:"results"`
+}
+
+type testResult struct {
+	File   string `json:"file"`
+	Name   string `json:"name"`
+	Passed bool   `json:"passed"`
+	Reason string `json:"reason,omitempty"`
+	Got    string `json:"got,omitempty"`
+	Want   string `json:"want,omitempty"`
+}
+
+// runTests runs the test cases in the given files or directories.
+func runTests(args []string, flags *core.CLIFlags) error {
+	paths, err := testsuite.Find(args)
+	if err != nil {
+		return core.NewE100("test", err)
+	} else if len(paths) == 0 {
+		return core.NewE100("test", errors.New("no test files found"))
+	}
+
+	runner := testsuite.NewRunner(flags)
+
+	var results []testsuite.Result
+	for _, path := range paths {
+		cases, lErr := testsuite.Load(path)
+		if lErr != nil {
+			return core.NewE100("test", lErr)
+		}
+
+		for _, c := range cases {
+			results = append(results, runner.Run(c))
+		}
+	}
+
+	// A case that could not be run at all is a broken configuration, not a
+	// failing assertion: report it as Vale's own error and stop.
+	for _, r := range results {
+		if r.Err != nil {
+			return core.NewE100(filepath.Base(r.Case.Path), r.Err)
+		}
+	}
+
+	if flags.Output == "JSON" {
+		return reportTestsJSON(results)
+	}
+
+	return reportTests(results, len(paths))
+}
+
+func reportTests(results []testsuite.Result, files int) error {
+	failed := 0
+
+	for _, r := range results {
+		if !r.Failed() {
+			continue
+		}
+		failed++
+
+		fmt.Printf("\n%s %s %s\n\n",
+			pterm.Red("✗"),
+			pterm.Bold.Sprint(r.Case.Name),
+			pterm.Gray("— "+r.Reason))
+
+		if r.Case.About != "" {
+			fmt.Printf("  %s %s\n", pterm.Gray("about"), r.Case.About)
+		}
+		fmt.Printf("  %s  %s\n\n", pterm.Gray("from"), relPath(r.Case.Path))
+
+		if r.Case.Want != nil {
+			fmt.Print(renderDiff(testsuite.Diff(*r.Case.Want, r.Got)))
+		} else {
+			fmt.Print(indentBlock(blockOrNone(r.Got)))
+		}
+	}
+
+	summary := fmt.Sprintf("%d %s — %d passed, %d failed",
+		files, pluralize("file", files), len(results)-failed, failed)
+
+	if failed > 0 {
+		fmt.Println()
+		pterm.Error.Println(summary)
+		return errTestFailed
+	}
+
+	pterm.Success.Println(summary)
+	return nil
+}
+
+func reportTestsJSON(results []testsuite.Result) error {
+	report := testReport{Results: make([]testResult, 0, len(results))}
+
+	for _, r := range results {
+		if r.Failed() {
+			report.Failed++
+		} else {
+			report.Passed++
+		}
+
+		out := testResult{
+			File:   r.Case.Path,
+			Name:   r.Case.Name,
+			Passed: !r.Failed(),
+			Reason: r.Reason,
+			Got:    r.Got,
+		}
+		if r.Case.Want != nil {
+			out.Want = *r.Case.Want
+		}
+
+		report.Results = append(report.Results, out)
+	}
+
+	if err := printJSON(report); err != nil {
+		return err
+	}
+	if report.Failed > 0 {
+		return errTestFailed
+	}
+
+	return nil
+}
+
+// renderDiff colours a diff and indents it under its heading.
+func renderDiff(diff []testsuite.Line) string {
+	if len(diff) == 0 {
+		return indentBlock("(no alerts)")
+	}
+
+	var b strings.Builder
+	for _, line := range diff {
+		text := fmt.Sprintf("%c %s", line.Op, line.Text)
+
+		switch line.Op {
+		case testsuite.Del:
+			text = pterm.Red(text)
+		case testsuite.Add:
+			text = pterm.Green(text)
+		case testsuite.Same:
+			text = pterm.Gray(text)
+		}
+
+		fmt.Fprintf(&b, "    %s\n", text)
+	}
+
+	fmt.Fprintf(&b, "\n    %s   %s\n",
+		pterm.Red("- expected"), pterm.Green("+ actual"))
+
+	return b.String()
+}
+
+// relPath shortens a path against the working directory, since that is where
+// the reader is standing.
+func relPath(path string) string {
+	wd, err := os.Getwd()
+	if err != nil {
+		return path
+	}
+
+	rel, err := filepath.Rel(wd, path)
+	if err != nil || strings.HasPrefix(rel, "..") {
+		return path
+	}
+
+	return rel
+}
+
+// indentBlock indents a rendered block so it sits under its heading.
+func indentBlock(s string) string {
+	var b strings.Builder
+	for _, line := range strings.Split(strings.TrimRight(s, "\n"), "\n") {
+		fmt.Fprintf(&b, "    %s\n", line)
+	}
+	return b.String()
+}
+
+func blockOrNone(s string) string {
+	if strings.TrimSpace(s) == "" {
+		return "(no alerts)"
+	}
+	return s
+}

internal/check/manager.go

@@ -156,9 +156,15 @@ func (mgr *Manager) addStyle(path string) error {
 
 	var sources []source
 	err := system.Walk(path, func(fp string, info fs.FileInfo, err error) error {
-		if err != nil {
+		switch {
+		case err != nil:
 			return err
-		} else if info.IsDir() || !strings.HasSuffix(info.Name(), ".yml") {
+		case info.IsDir() || !strings.HasSuffix(info.Name(), ".yml"):
+			return nil
+		case core.IsTestFile(info.Name()):
+			// A rule's cases live beside it, so the style directory holds YAML
+			// that is not a rule. Loaded as one it fails on `extends`, and the
+			// whole configuration stops. See #1122.
 			return nil
 		}
 		sources = append(sources, source{name: info.Name(), path: fp})

internal/core/format.go

@@ -6,6 +6,26 @@ import (
 	"strings"
 )
 
+// TestFileSuffixes name the YAML that holds a rule's test cases rather than a
+// rule. `.yml` is the spelling Vale uses elsewhere and the one to document;
+// `.yaml` is accepted so that a configuration written the other way still
+// works.
+//
+// A rule's cases live beside it, which puts them inside the StylesPath: the
+// loader that has to skip them and the runner that has to find them must agree
+// on which is which. See #1122.
+var TestFileSuffixes = []string{".test.yml", ".test.yaml"}
+
+// IsTestFile reports whether a file name holds test cases rather than a rule.
+func IsTestFile(name string) bool {
+	for _, suffix := range TestFileSuffixes {
+		if strings.HasSuffix(name, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
 // CommentsByNormedExt determines what parts of a file we should lint -- e.g.,
 // we only want to lint ; comments in a Clojure file.
 //

internal/e2e/report_test.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/errata-ai/vale/v3/internal/testsuite"
 )
 
 // Colors are used only when the output is a terminal that asked for them.
@@ -211,30 +213,20 @@ func report(t *testing.T, s *suite, tc testCase, reason, body string) {
 }
 
 // diffLines renders a line diff of want against got.
+//
+// The comparison is testsuite's, which `vale test` uses for the same job on
+// the same kind of output. Only the colouring is this suite's own.
 func diffLines(want, got string) string {
-	w, g := lines(want), lines(got)
-	common := lcs(w, g)
-
 	var b strings.Builder
-	var i, j int
 
-	for _, c := range common {
-		for i < len(w) && w[i] != c {
-			fmt.Fprintf(&b, "  %s- %s%s\n", red, w[i], reset)
-			i++
-		}
-		for j < len(g) && g[j] != c {
-			fmt.Fprintf(&b, "  %s+ %s%s\n", green, g[j], reset)
-			j++
+	for _, line := range testsuite.Diff(want, got) {
+		color := dim
+		if line.Op == testsuite.Del {
+			color = red
+		} else if line.Op == testsuite.Add {
+			color = green
 		}
-		fmt.Fprintf(&b, "  %s  %s%s\n", dim, c, reset)
-		i, j = i+1, j+1
-	}
-	for ; i < len(w); i++ {
-		fmt.Fprintf(&b, "  %s- %s%s\n", red, w[i], reset)
-	}
-	for ; j < len(g); j++ {
-		fmt.Fprintf(&b, "  %s+ %s%s\n", green, g[j], reset)
+		fmt.Fprintf(&b, "  %s%c %s%s\n", color, line.Op, line.Text, reset)
 	}
 
 	fmt.Fprintf(&b, "\n  %s%s- expected   %s+ actual%s\n", dim, red, green, reset)
@@ -263,39 +255,6 @@ func lines(s string) []string {
 	return strings.Split(s, "\n")
 }
 
-// lcs returns the longest common subsequence of a and b.
-func lcs(a, b []string) []string {
-	n := make([][]int, len(a)+1)
-	for i := range n {
-		n[i] = make([]int, len(b)+1)
-	}
-
-	for i := len(a) - 1; i >= 0; i-- {
-		for j := len(b) - 1; j >= 0; j-- {
-			if a[i] == b[j] {
-				n[i][j] = n[i+1][j+1] + 1
-			} else {
-				n[i][j] = max(n[i+1][j], n[i][j+1])
-			}
-		}
-	}
-
-	var out []string
-	for i, j := 0, 0; i < len(a) && j < len(b); {
-		switch {
-		case a[i] == b[j]:
-			out = append(out, a[i])
-			i, j = i+1, j+1
-		case n[i+1][j] >= n[i][j+1]:
-			i++
-		default:
-			j++
-		}
-	}
-
-	return out
-}
-
 // rel shortens a path for display, relative to the repository root.
 func rel(path string) string {
 	root := filepath.Dir(testdata)