From 79c1766176d23f9fc945fcb96e06544ea7debce4 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Tue, 23 Dec 2025 11:47:47 +0100 Subject: [PATCH 01/10] Outline KQL package --- klog/service/kql/parse.go | 135 +++++++++++++++++++ klog/service/kql/parse_test.go | 212 ++++++++++++++++++++++++++++++ klog/service/kql/predicate.go | 108 +++++++++++++++ klog/service/kql/query.go | 23 ++++ klog/service/kql/query_test.go | 155 ++++++++++++++++++++++ klog/service/kql/tokenise.go | 110 ++++++++++++++++ klog/service/kql/tokenise_test.go | 120 +++++++++++++++++ klog/service/kql/util.go | 136 +++++++++++++++++++ 8 files changed, 999 insertions(+) create mode 100644 klog/service/kql/parse.go create mode 100644 klog/service/kql/parse_test.go create mode 100644 klog/service/kql/predicate.go create mode 100644 klog/service/kql/query.go create mode 100644 klog/service/kql/query_test.go create mode 100644 klog/service/kql/tokenise.go create mode 100644 klog/service/kql/tokenise_test.go create mode 100644 klog/service/kql/util.go diff --git a/klog/service/kql/parse.go b/klog/service/kql/parse.go new file mode 100644 index 0000000..ab3d87e --- /dev/null +++ b/klog/service/kql/parse.go @@ -0,0 +1,135 @@ +package kql + +import ( + "errors" + "fmt" + + "github.com/jotaen/klog/klog" + "github.com/jotaen/klog/klog/service/period" +) + +var ( + ErrMalformedQuery = errors.New("Malformed query") // This is only a just-in-case fallback. + ErrCannotMixAndOr = errors.New("Cannot mix && and || operators on the same level. Please use parenthesis () for grouping.") + ErrUnbalancedBrackets = errors.New("Unbalanced parenthesis. Please make sure that the number of opening and closing parentheses matches.") + errOperatorOperand = errors.New("Missing expected") // Internal “base” class + ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) + ErrOperandExpected = fmt.Errorf("%w operand. Please remove redundant logical operators.", errOperatorOperand) + ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") +) + +func Parse(query string) (Predicate, error) { + tokens, err := tokenise(query) + if err != nil { + return nil, err + } + tp := newTokenParser(append(tokens, tokenCloseBracket{})) + p, err := parseGroup(&tp) + if err != nil { + return nil, err + } + // Check whether there are tokens left, which would indicate + // unbalanced brackets. + if tp.next() != nil { + return nil, ErrUnbalancedBrackets + } + return p, nil +} + +func parseGroup(tp *tokenParser) (Predicate, error) { + g := predicateGroup{} + + for { + nextToken := tp.next() + if nextToken == nil { + break + } + + switch tk := nextToken.(type) { + + case tokenOpenBracket: + if err := tp.checkNextIsOperand(); err != nil { + return nil, err + } + p, err := parseGroup(tp) + if err != nil { + return nil, err + } + g.append(p) + + case tokenCloseBracket: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + p, err := g.make() + return p, err + + case tokenDate: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + date, err := klog.NewDateFromString(tk.date) + if err != nil { + return nil, err + } + g.append(IsInDateRange{date, date}) + + case tokenDateRange: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + dateBoundaries := []klog.Date{nil, nil} + for i, v := range tk.bounds { + if v == "" { + continue + } + date, err := klog.NewDateFromString(v) + if err != nil { + return nil, err + } + dateBoundaries[i] = date + } + g.append(IsInDateRange{dateBoundaries[0], dateBoundaries[1]}) + + case tokenPeriod: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + prd, err := period.NewPeriodFromPatternString(tk.period) + if err != nil { + return nil, err + } + g.append(IsInDateRange{prd.Since(), prd.Until()}) + + case tokenAnd, tokenOr: + if err := tp.checkNextIsOperand(); err != nil { + return nil, err + } + err := g.setOperator(tk) + if err != nil { + return nil, err + } + + case tokenNot: + if err := tp.checkNextIsOperand(); err != nil { + return nil, err + } + g.negateNextOperand() + + case tokenTag: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + tag, err := klog.NewTagFromString(tk.tag) + if err != nil { + return nil, err + } + g.append(HasTag{tag}) + + default: + panic("Unrecognized token") + } + } + + return nil, ErrUnbalancedBrackets +} diff --git a/klog/service/kql/parse_test.go b/klog/service/kql/parse_test.go new file mode 100644 index 0000000..8edca2e --- /dev/null +++ b/klog/service/kql/parse_test.go @@ -0,0 +1,212 @@ +package kql + +import ( + "testing" + + "github.com/jotaen/klog/klog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAtDate(t *testing.T) { + p, err := Parse("2020-03-01") + require.Nil(t, err) + assert.Equal(t, + IsInDateRange{klog.Ɀ_Date_(2020, 3, 1), klog.Ɀ_Date_(2020, 3, 1)}, + p) +} + +func TestAndOperator(t *testing.T) { + p, err := Parse("2020-01-01 && #hello") + require.Nil(t, err) + assert.Equal(t, + And{[]Predicate{ + IsInDateRange{klog.Ɀ_Date_(2020, 1, 1), klog.Ɀ_Date_(2020, 1, 1)}, + HasTag{klog.NewTagOrPanic("hello", "")}, + }}, p) +} + +func TestOrOperator(t *testing.T) { + p, err := Parse("#foo || 1999-12-31") + require.Nil(t, err) + assert.Equal(t, + Or{[]Predicate{ + HasTag{klog.NewTagOrPanic("foo", "")}, + IsInDateRange{klog.Ɀ_Date_(1999, 12, 31), klog.Ɀ_Date_(1999, 12, 31)}, + }}, p) +} + +func TestCannotMixAndOrOnSameLevel(t *testing.T) { + for _, tt := range []string{ + "#foo || 1999-12-31 && 2000-01-02", + "#foo || 1999-12-31 || 2000-01-02 && 2020-07-24", + "#foo && 1999-12-31 || 2000-01-02", + "#foo && 1999-12-31 && 2000-01-02 || 2020-07-24", + "#foo && (1999-12-31 || 2000-01-02 && 2021-05-17)", + "#foo || (1999-12-31 && 2000-01-02 || 2021-05-17)", + } { + t.Run(tt, func(t *testing.T) { + p, err := Parse(tt) + require.ErrorIs(t, err, ErrCannotMixAndOr) + require.Nil(t, p) + }) + } +} + +func TestNotOperator(t *testing.T) { + p, err := Parse("!2020-01-01 && !#hello && !(2021-04-05 || #foo)") + require.Nil(t, err) + assert.Equal(t, + And{[]Predicate{ + Not{ + IsInDateRange{klog.Ɀ_Date_(2020, 1, 1), klog.Ɀ_Date_(2020, 1, 1)}, + }, + Not{ + HasTag{klog.NewTagOrPanic("hello", "")}, + }, + Not{ + Or{[]Predicate{ + IsInDateRange{klog.Ɀ_Date_(2021, 4, 5), klog.Ɀ_Date_(2021, 4, 5)}, + HasTag{klog.NewTagOrPanic("foo", "")}, + }}, + }, + }}, p) +} + +func TestGrouping(t *testing.T) { + p, err := Parse("(#foo || #bar || #xyz) && 1999-12-31") + require.Nil(t, err) + assert.Equal(t, + And{[]Predicate{ + Or{[]Predicate{ + HasTag{klog.NewTagOrPanic("foo", "")}, + HasTag{klog.NewTagOrPanic("bar", "")}, + HasTag{klog.NewTagOrPanic("xyz", "")}, + }}, + IsInDateRange{klog.Ɀ_Date_(1999, 12, 31), klog.Ɀ_Date_(1999, 12, 31)}, + }}, p) +} + +func TestNestedGrouping(t *testing.T) { + p, err := Parse("((#foo && (#bar || #xyz)) && 1999-12-31) || 1970-03-12") + require.Nil(t, err) + assert.Equal(t, + Or{[]Predicate{ + And{[]Predicate{ + And{[]Predicate{ + HasTag{klog.NewTagOrPanic("foo", "")}, + Or{[]Predicate{ + HasTag{klog.NewTagOrPanic("bar", "")}, + HasTag{klog.NewTagOrPanic("xyz", "")}, + }}, + }}, + IsInDateRange{klog.Ɀ_Date_(1999, 12, 31), klog.Ɀ_Date_(1999, 12, 31)}, + }}, + IsInDateRange{klog.Ɀ_Date_(1970, 3, 12), klog.Ɀ_Date_(1970, 03, 12)}, + }}, p) +} + +func TestClosedDateRange(t *testing.T) { + p, err := Parse("2020-03-06...2020-04-22") + require.Nil(t, err) + assert.Equal(t, + IsInDateRange{klog.Ɀ_Date_(2020, 3, 6), klog.Ɀ_Date_(2020, 4, 22)}, + p) +} + +func TestOpenDateRangeSince(t *testing.T) { + p, err := Parse("2020-03-01...") + require.Nil(t, err) + assert.Equal(t, + IsInDateRange{klog.Ɀ_Date_(2020, 3, 1), nil}, + p) +} + +func TestOpenDateRangeUntil(t *testing.T) { + p, err := Parse("...2020-03-01") + require.Nil(t, err) + assert.Equal(t, + IsInDateRange{nil, klog.Ɀ_Date_(2020, 3, 1)}, + p) +} + +func TestPeriod(t *testing.T) { + p, err := Parse("2020 || 2021-Q2 || 2022-08 || 2023-W46") + require.Nil(t, err) + assert.Equal(t, + Or{[]Predicate{ + IsInDateRange{klog.Ɀ_Date_(2020, 1, 1), klog.Ɀ_Date_(2020, 12, 31)}, + IsInDateRange{klog.Ɀ_Date_(2021, 4, 1), klog.Ɀ_Date_(2021, 6, 30)}, + IsInDateRange{klog.Ɀ_Date_(2022, 8, 1), klog.Ɀ_Date_(2022, 8, 31)}, + IsInDateRange{klog.Ɀ_Date_(2023, 11, 13), klog.Ɀ_Date_(2023, 11, 19)}, + }}, p) +} + +func TestTags(t *testing.T) { + p, err := Parse("#tag || #tag-with=value || #tag-with='quoted value'") + require.Nil(t, err) + assert.Equal(t, + Or{[]Predicate{ + HasTag{klog.NewTagOrPanic("tag", "")}, + HasTag{klog.NewTagOrPanic("tag-with", "value")}, + HasTag{klog.NewTagOrPanic("tag-with", "quoted value")}, + }}, p) +} + +func TestBracketMismatch(t *testing.T) { + for _, tt := range []string{ + "(2020-01", + "((2020-01", + "(2020-01-01))", + "2020-01-01)", + "(2020-01-01 && (2020-02-02 || 2020-03-03", + "(2020-01-01 && (2020-02-02))) || 2020-03-03", + } { + t.Run(tt, func(t *testing.T) { + p, err := Parse(tt) + require.ErrorIs(t, err, ErrUnbalancedBrackets) + require.Nil(t, p) + }) + } +} + +func TestOperatorOperandSequence(t *testing.T) { + for _, tt := range []string{ + // Operands: (date, date-range, period, tag) + "2020-01-01 2020-02-02", + "2020-01-01 (#foo && #bar)", + "(#foo && #bar) 2020-01-01", + "(#foo && #bar) #foo", + "2020-01-01...2020-02-28 #foo", + "2020-01-01... #foo", + "...2020-01-01 #foo", + "2020-01 2020-02", + "2020-01-01 #foo", + "2020-01 #foo", + "#foo 2020-01-01", + "#foo 2020-01", + "#foo #foo", + + // And: + "2020-01-01 && #tag #foo", + "2020-01-01 && && 2020-02-02", + "2020-01-01 && ( && #foo)", + + // Or: + "2020-01-01 || #tag #foo", + "2020-01-01 || || 2020-02-02", + "2020-01-01 && ( || #foo)", + + // Not: + "!&& #foo", + "!|| #foo", + "(!) #foo", + "#foo !", + } { + t.Run(tt, func(t *testing.T) { + p, err := Parse(tt) + require.ErrorIs(t, err, errOperatorOperand) + require.Nil(t, p) + }) + } +} diff --git a/klog/service/kql/predicate.go b/klog/service/kql/predicate.go new file mode 100644 index 0000000..0cab0f8 --- /dev/null +++ b/klog/service/kql/predicate.go @@ -0,0 +1,108 @@ +package kql + +import "github.com/jotaen/klog/klog" + +type queriedEntry struct { + parent klog.Record + entry klog.Entry +} + +type Predicate interface { + Matches(queriedEntry) bool +} + +type IsInDateRange struct { + From klog.Date + To klog.Date +} + +func (i IsInDateRange) Matches(e queriedEntry) bool { + isAfter := func() bool { + if i.From == nil { + return true + } + return e.parent.Date().IsAfterOrEqual(i.From) + }() + isBefore := func() bool { + if i.To == nil { + return true + } + return i.To.IsAfterOrEqual(e.parent.Date()) + }() + return isAfter && isBefore +} + +type HasTag struct { + Tag klog.Tag +} + +func (h HasTag) Matches(e queriedEntry) bool { + return e.parent.Summary().Tags().Contains(h.Tag) || e.entry.Summary().Tags().Contains(h.Tag) +} + +type And struct { + Predicates []Predicate +} + +func (a And) Matches(e queriedEntry) bool { + for _, p := range a.Predicates { + if !p.Matches(e) { + return false + } + } + return true +} + +type Or struct { + Predicates []Predicate +} + +func (o Or) Matches(e queriedEntry) bool { + for _, p := range o.Predicates { + if p.Matches(e) { + return true + } + } + return false +} + +type Not struct { + Predicate Predicate +} + +func (n Not) Matches(e queriedEntry) bool { + return !n.Predicate.Matches(e) +} + +type EntryType string + +const ( + ENTRY_TYPE_DURATION = EntryType("DURATION") + ENTRY_TYPE_POSITIVE_DURATION = EntryType("DURATION_POSITIVE") + ENTRY_TYPE_NEGATIVE_DURATION = EntryType("DURATION_NEGATIVE") + ENTRY_TYPE_RANGE = EntryType("RANGE") + ENTRY_TYPE_OPEN_RANGE = EntryType("OPEN_RANGE") +) + +type IsEntryType struct { + Type EntryType +} + +func (t IsEntryType) Matches(e queriedEntry) bool { + return klog.Unbox[bool](&e.entry, func(r klog.Range) bool { + return t.Type == ENTRY_TYPE_RANGE + }, func(duration klog.Duration) bool { + if t.Type == ENTRY_TYPE_DURATION { + return true + } + if t.Type == ENTRY_TYPE_POSITIVE_DURATION && e.entry.Duration().InMinutes() >= 0 { + return true + } + if t.Type == ENTRY_TYPE_NEGATIVE_DURATION && e.entry.Duration().InMinutes() < 0 { + return true + } + return false + }, func(openRange klog.OpenRange) bool { + return t.Type == ENTRY_TYPE_OPEN_RANGE + }) +} diff --git a/klog/service/kql/query.go b/klog/service/kql/query.go new file mode 100644 index 0000000..e19088a --- /dev/null +++ b/klog/service/kql/query.go @@ -0,0 +1,23 @@ +package kql + +import ( + "github.com/jotaen/klog/klog" +) + +func Query(p Predicate, rs []klog.Record) []klog.Record { + var res []klog.Record + for _, r := range rs { + var es []klog.Entry + for i, e := range r.Entries() { + if p.Matches(queriedEntry{r, r.Entries()[i]}) { + es = append(es, e) + } + } + if len(es) == 0 { + continue + } + r.SetEntries(es) + res = append(res, r) + } + return res +} diff --git a/klog/service/kql/query_test.go b/klog/service/kql/query_test.go new file mode 100644 index 0000000..f14c3bf --- /dev/null +++ b/klog/service/kql/query_test.go @@ -0,0 +1,155 @@ +package kql + +import ( + "testing" + + "github.com/jotaen/klog/klog" + "github.com/jotaen/klog/klog/service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func sampleRecordsForQuerying() []klog.Record { + return []klog.Record{ + func() klog.Record { + // Note that records without entries never match any query. + r := klog.NewRecord(klog.Ɀ_Date_(1999, 12, 30)) + r.SetSummary(klog.Ɀ_RecordSummary_("Hello World", "#foo")) + return r + }(), func() klog.Record { + r := klog.NewRecord(klog.Ɀ_Date_(1999, 12, 31)) + r.AddDuration(klog.NewDuration(5, 0), klog.Ɀ_EntrySummary_("#bar")) + return r + }(), func() klog.Record { + r := klog.NewRecord(klog.Ɀ_Date_(2000, 1, 1)) + r.SetSummary(klog.Ɀ_RecordSummary_("#foo")) + r.AddDuration(klog.NewDuration(0, 15), nil) + r.AddDuration(klog.NewDuration(6, 0), klog.Ɀ_EntrySummary_("#bar")) + r.AddDuration(klog.NewDuration(0, -30), nil) + return r + }(), func() klog.Record { + r := klog.NewRecord(klog.Ɀ_Date_(2000, 1, 2)) + r.SetSummary(klog.Ɀ_RecordSummary_("#foo")) + r.AddDuration(klog.NewDuration(7, 0), nil) + return r + }(), func() klog.Record { + r := klog.NewRecord(klog.Ɀ_Date_(2000, 1, 3)) + r.SetSummary(klog.Ɀ_RecordSummary_("#foo=a")) + r.AddDuration(klog.NewDuration(4, 0), klog.Ɀ_EntrySummary_("test", "foo #bar=1")) + r.AddDuration(klog.NewDuration(4, 0), klog.Ɀ_EntrySummary_("#bar=2")) + r.Start(klog.NewOpenRange(klog.Ɀ_Time_(12, 00)), nil) + return r + }(), + } +} + +func TestQueryWithNoClauses(t *testing.T) { + rs := Query(And{}, sampleRecordsForQuerying()) + require.Len(t, rs, 4) + assert.Equal(t, klog.NewDuration(5+6+7+8, -30+15), service.Total(rs...)) +} + +func TestQueryWithAtDate(t *testing.T) { + rs := Query(IsInDateRange{ + From: klog.Ɀ_Date_(2000, 1, 2), + To: klog.Ɀ_Date_(2000, 1, 2), + }, sampleRecordsForQuerying()) + require.Len(t, rs, 1) + assert.Equal(t, klog.NewDuration(7, 0), service.Total(rs...)) +} + +func TestQueryWithAfter(t *testing.T) { + rs := Query(IsInDateRange{ + From: klog.Ɀ_Date_(2000, 1, 1), + To: nil, + }, sampleRecordsForQuerying()) + require.Len(t, rs, 3) + assert.Equal(t, 1, rs[0].Date().Day()) + assert.Equal(t, 2, rs[1].Date().Day()) + assert.Equal(t, 3, rs[2].Date().Day()) +} + +func TestQueryWithBefore(t *testing.T) { + rs := Query(IsInDateRange{ + From: nil, + To: klog.Ɀ_Date_(2000, 1, 1), + }, sampleRecordsForQuerying()) + require.Len(t, rs, 2) + assert.Equal(t, 31, rs[0].Date().Day()) + assert.Equal(t, 1, rs[1].Date().Day()) +} + +//func TestQueryWithTagOnEntries(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "")}}) +// require.Len(t, rs, 3) +// assert.Equal(t, 31, rs[0].Date().Day()) +// assert.Equal(t, 1, rs[1].Date().Day()) +// assert.Equal(t, 3, rs[2].Date().Day()) +// assert.Equal(t, klog.NewDuration(5+8+6, 0), Total(rs...)) +//} +// +//func TestQueryWithTagOnOverallSummary(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "")}}) +// require.Len(t, rs, 4) +// assert.Equal(t, 30, rs[0].Date().Day()) +// assert.Equal(t, 1, rs[1].Date().Day()) +// assert.Equal(t, 2, rs[2].Date().Day()) +// assert.Equal(t, 3, rs[3].Date().Day()) +// assert.Equal(t, klog.NewDuration(6+7+8, -30+15), Total(rs...)) +//} +// +//func TestQueryWithTagOnEntriesAndInSummary(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", ""), klog.NewTagOrPanic("bar", "")}}) +// require.Len(t, rs, 2) +// assert.Equal(t, 1, rs[0].Date().Day()) +// assert.Equal(t, 3, rs[1].Date().Day()) +// assert.Equal(t, klog.NewDuration(8+6, 0), Total(rs...)) +//} +// +//func TestQueryWithTagValues(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "a")}}) +// require.Len(t, rs, 1) +// assert.Equal(t, 3, rs[0].Date().Day()) +// assert.Equal(t, klog.NewDuration(8, 0), Total(rs...)) +//} +// +//func TestQueryWithTagValuesInEntries(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "1")}}) +// require.Len(t, rs, 1) +// assert.Equal(t, 3, rs[0].Date().Day()) +// assert.Equal(t, klog.NewDuration(4, 0), Total(rs...)) +//} +// +//func TestQueryWithTagNonMatchingValues(t *testing.T) { +// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "3")}}) +// require.Len(t, rs, 0) +//} +// +//func TestQueryWithEntryTypes(t *testing.T) { +// { +// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_DURATION}) +// require.Len(t, rs, 4) +// assert.Equal(t, klog.NewDuration(0, 1545), Total(rs...)) +// } +// { +// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_NEGATIVE_DURATION}) +// require.Len(t, rs, 1) +// assert.Equal(t, 1, rs[0].Date().Day()) +// assert.Equal(t, klog.NewDuration(0, -30), Total(rs...)) +// } +// { +// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_POSITIVE_DURATION}) +// require.Len(t, rs, 4) +// assert.Equal(t, klog.NewDuration(0, 1575), Total(rs...)) +// } +// { +// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_RANGE}) +// require.Len(t, rs, 0) +// assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) +// } +// { +// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_OPEN_RANGE}) +// require.Len(t, rs, 1) +// assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) +// } +//} diff --git a/klog/service/kql/tokenise.go b/klog/service/kql/tokenise.go new file mode 100644 index 0000000..59e9cc6 --- /dev/null +++ b/klog/service/kql/tokenise.go @@ -0,0 +1,110 @@ +package kql + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +type token interface{} + +type tokenOpenBracket struct{} +type tokenCloseBracket struct{} +type tokenAnd struct{} +type tokenOr struct{} +type tokenNot struct{} +type tokenDate struct { + date string +} +type tokenPeriod struct { + period string +} +type tokenDateRange struct { + bounds []string +} +type tokenTag struct { + tag string +} + +var ( + tagRegex = regexp.MustCompile(`^#(([\p{L}\d_-]+)(=(("[^"]*")|('[^']*')|([\p{L}\d_-]*)))?)`) + dateRangeRegex = regexp.MustCompile(`^((\d{4}-\d{2}-\d{2})?\.\.\.(\d{4}-\d{2}-\d{2})?)`) + dateRegex = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})`) + periodRegex = regexp.MustCompile(`^((\d{4}-\p{L}?\d+)|(\d{4}))`) +) + +var ( + ErrMissingWhiteSpace = errors.New("Missing whitespace. Please separate operands and operators with whitespace.") + ErrUnrecognisedToken = errors.New("Unrecognised query token. Please make sure to use valid query syntax.") +) + +func tokenise(query string) ([]token, error) { + txtParser := newTextParser(query) + tokens := []token{} + for { + if txtParser.isFinished() { + break + } + + if txtParser.peekString(" ") { + txtParser.advance(1) + } else if txtParser.peekString("(") { + tokens = append(tokens, tokenOpenBracket{}) + txtParser.advance(1) + } else if txtParser.peekString(")") { + tokens = append(tokens, tokenCloseBracket{}) + txtParser.advance(1) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } + } else if txtParser.peekString("&&") { + tokens = append(tokens, tokenAnd{}) + txtParser.advance(2) + if !txtParser.peekString(EOT, " ") { + return nil, ErrMissingWhiteSpace + } + } else if txtParser.peekString("||") { + tokens = append(tokens, tokenOr{}) + txtParser.advance(2) + if !txtParser.peekString(EOT, " ") { + return nil, ErrMissingWhiteSpace + } + } else if txtParser.peekString("!") { + tokens = append(tokens, tokenNot{}) + txtParser.advance(1) + } else if tm := txtParser.peekRegex(tagRegex); tm != nil { + value := tm[1] + tokens = append(tokens, tokenTag{value}) + txtParser.advance(1 + len(value)) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } + } else if rm := txtParser.peekRegex(dateRangeRegex); rm != nil { + value := rm[1] + parts := strings.Split(value, "...") + tokens = append(tokens, tokenDateRange{parts}) + txtParser.advance(len(value)) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } + } else if dm := txtParser.peekRegex(dateRegex); dm != nil { + value := dm[1] + tokens = append(tokens, tokenDate{value}) + txtParser.advance(len(value)) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } + } else if pm := txtParser.peekRegex(periodRegex); pm != nil { + value := pm[1] + tokens = append(tokens, tokenPeriod{value}) + txtParser.advance(len(value)) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } + } else { + return nil, fmt.Errorf("%w: %s", ErrUnrecognisedToken, txtParser.remainder()) + } + } + return tokens, nil +} diff --git a/klog/service/kql/tokenise_test.go b/klog/service/kql/tokenise_test.go new file mode 100644 index 0000000..1123167 --- /dev/null +++ b/klog/service/kql/tokenise_test.go @@ -0,0 +1,120 @@ +package kql + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokeniseEmptyToken(t *testing.T) { + { // Empty + p, err := tokenise("") + require.Nil(t, err) + assert.Equal(t, p, []token{}) + } + { // Blank + p, err := tokenise(" ") + require.Nil(t, err) + assert.Equal(t, p, []token{}) + } +} + +func TestTokeniseAllTokens(t *testing.T) { + p, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4)") + require.Nil(t, err) + assert.Equal(t, []token{ + tokenDate{"2020-01-01"}, + tokenAnd{}, + tokenTag{"hello"}, + tokenOr{}, + tokenOpenBracket{}, + tokenDate{"2020-02-02"}, + tokenAnd{}, + tokenNot{}, + tokenPeriod{"2021-Q4"}, + tokenCloseBracket{}, + }, p) +} + +func TestDisregardWhitespaceBetweenTokens(t *testing.T) { + p, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) ") + require.Nil(t, err) + assert.Equal(t, []token{ + tokenDate{"2020-01-01"}, + tokenAnd{}, + tokenTag{"hello"}, + tokenOr{}, + tokenOpenBracket{}, + tokenDate{"2020-02-02"}, + tokenAnd{}, + tokenNot{}, + tokenPeriod{"2021-Q4"}, + tokenCloseBracket{}, + }, p) +} + +func TestFailsOnUnrecognisedToken(t *testing.T) { + for _, txt := range []string{ + "abcde", + "2020-01-01 & 2020-01-02", + "2020-01-01 * 2020-01-02", + "2020-01-01 {2020-01-02}", + } { + t.Run(txt, func(t *testing.T) { + p, err := tokenise(txt) + require.ErrorIs(t, err, ErrUnrecognisedToken) + assert.Nil(t, p) + }) + } +} + +func TestFailsOnMissingWhitespace(t *testing.T) { + for _, txt := range []string{ + "2021-12-12 &&&", + "2021-12-12 &&&&", + "2021-12-12 &&||", + "2021-12-12 &&2021-12-12", + "2021-12-12 &&#tag", + "2021-12-12 &&(2021-12-12 || #foo)", + + "2021-12-12 |||", + "2021-12-12 ||||", + "2021-12-12 ||&&", + "2021-12-12 ||2021-12-12", + "2021-12-12 ||#tag", + "2021-12-12 ||(2021-12-12 || #foo)", + + "(#foo)(#bar)", + "( #foo )( #bar )", + + "2020-01-01&&", + "2020-01-01||", + "2020-01-01( #foo )", + "2020-01-01#foo", + + "2020-01-01...2020-01-31&&", + "2020-01-01...2020-01-31( #foo )", + "2020-01-01...&&", + "2020-01-01...( #foo )", + + "(2021-12-12 || #foo)2020-01-01", + "(2021-12-12 || #foo)&& #foo", + "(2021-12-12 || #foo)#foo", + + "#tag&& #tag", + "#tag|| #tag", + "#tag( 2020-01-01)", + + "2020-Q4&&", + "2020-Q4||", + "2020-Q4( 2020-01-01 )", + "2020-Q4!( 2020-01-01 )", + } { + t.Run(txt, func(t *testing.T) { + p, err := tokenise(txt) + require.ErrorIs(t, err, ErrMissingWhiteSpace) + assert.Nil(t, p) + }) + } +} diff --git a/klog/service/kql/util.go b/klog/service/kql/util.go new file mode 100644 index 0000000..d27a816 --- /dev/null +++ b/klog/service/kql/util.go @@ -0,0 +1,136 @@ +package kql + +import ( + "regexp" + "strings" +) + +const EOT = "" // End of text + +type textParser struct { + text string + pointer int +} + +func newTextParser(text string) textParser { + return textParser{ + text: text, + pointer: 0, + } +} + +func (t *textParser) isFinished() bool { + return t.pointer == len(t.text) +} + +func (t *textParser) peekString(lookup ...string) bool { + r := t.remainder() + for _, l := range lookup { + if l == EOT { + if r == EOT { + return true + } + } else if strings.HasPrefix(r, l) { + return true + } + } + return false +} + +func (t *textParser) peekRegex(lookup *regexp.Regexp) []string { + return lookup.FindStringSubmatch(t.remainder()) +} + +func (t *textParser) advance(i int) { + t.pointer += i +} + +func (t *textParser) remainder() string { + if t.isFinished() { + return "" + } + return t.text[t.pointer:] +} + +type tokenParser struct { + tokens []token + pointer int +} + +func newTokenParser(ts []token) tokenParser { + return tokenParser{ + tokens: ts, + pointer: 0, + } +} + +func (t *tokenParser) next() token { + if t.pointer >= len(t.tokens) { + return nil + } + next := t.tokens[t.pointer] + t.pointer += 1 + return next +} + +func (t *tokenParser) checkNextIsOperand() error { + if t.pointer >= len(t.tokens) { + return ErrOperandExpected + } + switch t.tokens[t.pointer].(type) { + case tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot: + return nil + } + return ErrOperandExpected +} + +func (t *tokenParser) checkNextIsOperatorOrEnd() error { + if t.pointer >= len(t.tokens) { + return nil + } + switch t.tokens[t.pointer].(type) { + case tokenCloseBracket, tokenAnd, tokenOr: + return nil + } + return ErrOperatorExpected +} + +type predicateGroup struct { + ps []Predicate + operator token // nil or tokenAnd or tokenOr + isNextNegated bool +} + +func (g *predicateGroup) append(p Predicate) { + if g.isNextNegated { + g.isNextNegated = false + p = Not{p} + } + g.ps = append(g.ps, p) +} + +func (g *predicateGroup) setOperator(operatorT token) error { + if g.operator == nil { + g.operator = operatorT + } + if g.operator != operatorT { + return ErrCannotMixAndOr + } + return nil +} + +func (g *predicateGroup) negateNextOperand() { + g.isNextNegated = true +} + +func (g *predicateGroup) make() (Predicate, error) { + if len(g.ps) == 1 { + return g.ps[0], nil + } else if g.operator == (tokenAnd{}) { + return And{g.ps}, nil + } else if g.operator == (tokenOr{}) { + return Or{g.ps}, nil + } else { + return nil, ErrMalformedQuery + } +} From 64137a76615aae96fe00a2426080f6c77a71b786 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Tue, 23 Dec 2025 19:12:08 +0100 Subject: [PATCH 02/10] Add experimental `--query` filter flag --- klog/app/cli/args/filter.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/klog/app/cli/args/filter.go b/klog/app/cli/args/filter.go index f8825fd..ecc3363 100644 --- a/klog/app/cli/args/filter.go +++ b/klog/app/cli/args/filter.go @@ -6,6 +6,7 @@ import ( "github.com/jotaen/klog/klog" "github.com/jotaen/klog/klog/app" "github.com/jotaen/klog/klog/service" + "github.com/jotaen/klog/klog/service/kql" "github.com/jotaen/klog/klog/service/period" ) @@ -33,6 +34,8 @@ type FilterArgs struct { LastQuarter bool `hidden:"" name:"last-quarter" group:"Filter"` ThisYear bool `hidden:"" name:"this-year" group:"Filter"` LastYear bool `hidden:"" name:"last-year" group:"Filter"` + + Query string `name:"query" placeholder:"KQL-QUERY" group:"Filter" help:"Experimental."` } // FilterArgsCompletionOverrides enables/disables tab completion for @@ -51,6 +54,19 @@ var FilterArgsCompletionOverrides = map[string]bool{ } func (args *FilterArgs) ApplyFilter(now gotime.Time, rs []klog.Record) ([]klog.Record, app.Error) { + if args.Query != "" { + predicate, err := kql.Parse(args.Query) + if err != nil { + return nil, app.NewErrorWithCode( + app.GENERAL_ERROR, + "Malformed KQL Query", + err.Error(), + err, + ) + } + rs = kql.Query(predicate, rs) + return rs, nil + } today := klog.NewDateFromGo(now) qry := service.FilterQry{ BeforeOrEqual: args.Until, From 1cd11e273c13c698468483097344282a86420e42 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Tue, 23 Dec 2025 20:01:45 +0100 Subject: [PATCH 03/10] Add entry type filter --- klog/service/kql/parse.go | 10 ++++++++++ klog/service/kql/parse_test.go | 4 ++++ klog/service/kql/predicate.go | 32 +++++++++++++++++++++++++------ klog/service/kql/tokenise.go | 10 ++++++++++ klog/service/kql/tokenise_test.go | 8 ++++++-- klog/service/kql/util.go | 2 +- 6 files changed, 57 insertions(+), 9 deletions(-) diff --git a/klog/service/kql/parse.go b/klog/service/kql/parse.go index ab3d87e..9748ca9 100644 --- a/klog/service/kql/parse.go +++ b/klog/service/kql/parse.go @@ -116,6 +116,16 @@ func parseGroup(tp *tokenParser) (Predicate, error) { } g.negateNextOperand() + case tokenEntryType: + if err := tp.checkNextIsOperatorOrEnd(); err != nil { + return nil, err + } + et, err := NewEntryTypeFromString(tk.entryType) + if err != nil { + return nil, err + } + g.append(IsEntryType{et}) + case tokenTag: if err := tp.checkNextIsOperatorOrEnd(); err != nil { return nil, err diff --git a/klog/service/kql/parse_test.go b/klog/service/kql/parse_test.go index 8edca2e..f169540 100644 --- a/klog/service/kql/parse_test.go +++ b/klog/service/kql/parse_test.go @@ -186,6 +186,10 @@ func TestOperatorOperandSequence(t *testing.T) { "#foo 2020-01-01", "#foo 2020-01", "#foo #foo", + "type:duration #foo", + "#foo type:duration", + "2020 type:duration", + "type:duration 2025-Q4", // And: "2020-01-01 && #tag #foo", diff --git a/klog/service/kql/predicate.go b/klog/service/kql/predicate.go index 0cab0f8..2de1347 100644 --- a/klog/service/kql/predicate.go +++ b/klog/service/kql/predicate.go @@ -1,6 +1,11 @@ package kql -import "github.com/jotaen/klog/klog" +import ( + "errors" + "strings" + + "github.com/jotaen/klog/klog" +) type queriedEntry struct { parent klog.Record @@ -77,13 +82,28 @@ func (n Not) Matches(e queriedEntry) bool { type EntryType string const ( - ENTRY_TYPE_DURATION = EntryType("DURATION") - ENTRY_TYPE_POSITIVE_DURATION = EntryType("DURATION_POSITIVE") - ENTRY_TYPE_NEGATIVE_DURATION = EntryType("DURATION_NEGATIVE") - ENTRY_TYPE_RANGE = EntryType("RANGE") - ENTRY_TYPE_OPEN_RANGE = EntryType("OPEN_RANGE") + ENTRY_TYPE_DURATION = EntryType("duration") + ENTRY_TYPE_POSITIVE_DURATION = EntryType("duration-positive") + ENTRY_TYPE_NEGATIVE_DURATION = EntryType("duration-negative") + ENTRY_TYPE_RANGE = EntryType("range") + ENTRY_TYPE_OPEN_RANGE = EntryType("open-range") ) +func NewEntryTypeFromString(val string) (EntryType, error) { + for _, t := range []EntryType{ + ENTRY_TYPE_DURATION, + ENTRY_TYPE_POSITIVE_DURATION, + ENTRY_TYPE_NEGATIVE_DURATION, + ENTRY_TYPE_RANGE, + ENTRY_TYPE_OPEN_RANGE, + } { + if strings.ToLower(strings.ReplaceAll(val, "_", "-")) == string(t) { + return t, nil + } + } + return EntryType(""), errors.New("Illegal entry type") +} + type IsEntryType struct { Type EntryType } diff --git a/klog/service/kql/tokenise.go b/klog/service/kql/tokenise.go index 59e9cc6..0bc4c80 100644 --- a/klog/service/kql/tokenise.go +++ b/klog/service/kql/tokenise.go @@ -26,12 +26,16 @@ type tokenDateRange struct { type tokenTag struct { tag string } +type tokenEntryType struct { + entryType string +} var ( tagRegex = regexp.MustCompile(`^#(([\p{L}\d_-]+)(=(("[^"]*")|('[^']*')|([\p{L}\d_-]*)))?)`) dateRangeRegex = regexp.MustCompile(`^((\d{4}-\d{2}-\d{2})?\.\.\.(\d{4}-\d{2}-\d{2})?)`) dateRegex = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})`) periodRegex = regexp.MustCompile(`^((\d{4}-\p{L}?\d+)|(\d{4}))`) + typeRegex = regexp.MustCompile(`^type:([\p{L}\-_]+)`) ) var ( @@ -80,6 +84,12 @@ func tokenise(query string) ([]token, error) { if !txtParser.peekString(EOT, " ", ")") { return nil, ErrMissingWhiteSpace } + } else if ym := txtParser.peekRegex(typeRegex); ym != nil { + tokens = append(tokens, tokenEntryType{ym[1]}) + txtParser.advance(5 + len(ym[1])) + if !txtParser.peekString(EOT, " ", ")") { + return nil, ErrMissingWhiteSpace + } } else if rm := txtParser.peekRegex(dateRangeRegex); rm != nil { value := rm[1] parts := strings.Split(value, "...") diff --git a/klog/service/kql/tokenise_test.go b/klog/service/kql/tokenise_test.go index 1123167..374e172 100644 --- a/klog/service/kql/tokenise_test.go +++ b/klog/service/kql/tokenise_test.go @@ -21,7 +21,7 @@ func TestTokeniseEmptyToken(t *testing.T) { } func TestTokeniseAllTokens(t *testing.T) { - p, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4)") + p, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ tokenDate{"2020-01-01"}, @@ -34,11 +34,13 @@ func TestTokeniseAllTokens(t *testing.T) { tokenNot{}, tokenPeriod{"2021-Q4"}, tokenCloseBracket{}, + tokenAnd{}, + tokenEntryType{"duration"}, }, p) } func TestDisregardWhitespaceBetweenTokens(t *testing.T) { - p, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) ") + p, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ tokenDate{"2020-01-01"}, @@ -51,6 +53,8 @@ func TestDisregardWhitespaceBetweenTokens(t *testing.T) { tokenNot{}, tokenPeriod{"2021-Q4"}, tokenCloseBracket{}, + tokenAnd{}, + tokenEntryType{"duration"}, }, p) } diff --git a/klog/service/kql/util.go b/klog/service/kql/util.go index d27a816..b1c0422 100644 --- a/klog/service/kql/util.go +++ b/klog/service/kql/util.go @@ -78,7 +78,7 @@ func (t *tokenParser) checkNextIsOperand() error { return ErrOperandExpected } switch t.tokens[t.pointer].(type) { - case tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot: + case tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot, tokenEntryType: return nil } return ErrOperandExpected From fd300ee6cb6bc0283de01abf388aefdb702cadfc Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Sun, 28 Dec 2025 10:08:49 +0100 Subject: [PATCH 04/10] Rename package --- klog/app/cli/args/filter.go | 12 +++---- klog/service/{kql/query.go => kfl/filter.go} | 4 +-- .../{kql/query_test.go => kfl/filter_test.go} | 32 +++++++++---------- klog/service/{kql => kfl}/parse.go | 20 ++++++------ klog/service/{kql => kfl}/parse_test.go | 2 +- klog/service/{kql => kfl}/predicate.go | 2 +- klog/service/{kql => kfl}/tokenise.go | 6 ++-- klog/service/{kql => kfl}/tokenise_test.go | 2 +- klog/service/{kql => kfl}/util.go | 4 +-- 9 files changed, 42 insertions(+), 42 deletions(-) rename klog/service/{kql/query.go => kfl/filter.go} (82%) rename klog/service/{kql/query_test.go => kfl/filter_test.go} (76%) rename klog/service/{kql => kfl}/parse.go (74%) rename klog/service/{kql => kfl}/parse_test.go (99%) rename klog/service/{kql => kfl}/predicate.go (99%) rename klog/service/{kql => kfl}/tokenise.go (96%) rename klog/service/{kql => kfl}/tokenise_test.go (99%) rename klog/service/{kql => kfl}/util.go (97%) diff --git a/klog/app/cli/args/filter.go b/klog/app/cli/args/filter.go index ecc3363..d36a820 100644 --- a/klog/app/cli/args/filter.go +++ b/klog/app/cli/args/filter.go @@ -6,7 +6,7 @@ import ( "github.com/jotaen/klog/klog" "github.com/jotaen/klog/klog/app" "github.com/jotaen/klog/klog/service" - "github.com/jotaen/klog/klog/service/kql" + "github.com/jotaen/klog/klog/service/kfl" "github.com/jotaen/klog/klog/service/period" ) @@ -35,7 +35,7 @@ type FilterArgs struct { ThisYear bool `hidden:"" name:"this-year" group:"Filter"` LastYear bool `hidden:"" name:"last-year" group:"Filter"` - Query string `name:"query" placeholder:"KQL-QUERY" group:"Filter" help:"Experimental."` + FilterQuery string `name:"filter" placeholder:"KQL-FILTER-QUERY" group:"Filter" help:"(Experimental)"` } // FilterArgsCompletionOverrides enables/disables tab completion for @@ -54,17 +54,17 @@ var FilterArgsCompletionOverrides = map[string]bool{ } func (args *FilterArgs) ApplyFilter(now gotime.Time, rs []klog.Record) ([]klog.Record, app.Error) { - if args.Query != "" { - predicate, err := kql.Parse(args.Query) + if args.FilterQuery != "" { + predicate, err := kfl.Parse(args.FilterQuery) if err != nil { return nil, app.NewErrorWithCode( app.GENERAL_ERROR, - "Malformed KQL Query", + "Malformed filter query", err.Error(), err, ) } - rs = kql.Query(predicate, rs) + rs = kfl.Filter(predicate, rs) return rs, nil } today := klog.NewDateFromGo(now) diff --git a/klog/service/kql/query.go b/klog/service/kfl/filter.go similarity index 82% rename from klog/service/kql/query.go rename to klog/service/kfl/filter.go index e19088a..ca08686 100644 --- a/klog/service/kql/query.go +++ b/klog/service/kfl/filter.go @@ -1,10 +1,10 @@ -package kql +package kfl import ( "github.com/jotaen/klog/klog" ) -func Query(p Predicate, rs []klog.Record) []klog.Record { +func Filter(p Predicate, rs []klog.Record) []klog.Record { var res []klog.Record for _, r := range rs { var es []klog.Entry diff --git a/klog/service/kql/query_test.go b/klog/service/kfl/filter_test.go similarity index 76% rename from klog/service/kql/query_test.go rename to klog/service/kfl/filter_test.go index f14c3bf..797b399 100644 --- a/klog/service/kql/query_test.go +++ b/klog/service/kfl/filter_test.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "testing" @@ -44,13 +44,13 @@ func sampleRecordsForQuerying() []klog.Record { } func TestQueryWithNoClauses(t *testing.T) { - rs := Query(And{}, sampleRecordsForQuerying()) + rs := Filter(And{}, sampleRecordsForQuerying()) require.Len(t, rs, 4) assert.Equal(t, klog.NewDuration(5+6+7+8, -30+15), service.Total(rs...)) } func TestQueryWithAtDate(t *testing.T) { - rs := Query(IsInDateRange{ + rs := Filter(IsInDateRange{ From: klog.Ɀ_Date_(2000, 1, 2), To: klog.Ɀ_Date_(2000, 1, 2), }, sampleRecordsForQuerying()) @@ -59,7 +59,7 @@ func TestQueryWithAtDate(t *testing.T) { } func TestQueryWithAfter(t *testing.T) { - rs := Query(IsInDateRange{ + rs := Filter(IsInDateRange{ From: klog.Ɀ_Date_(2000, 1, 1), To: nil, }, sampleRecordsForQuerying()) @@ -70,7 +70,7 @@ func TestQueryWithAfter(t *testing.T) { } func TestQueryWithBefore(t *testing.T) { - rs := Query(IsInDateRange{ + rs := Filter(IsInDateRange{ From: nil, To: klog.Ɀ_Date_(2000, 1, 1), }, sampleRecordsForQuerying()) @@ -80,7 +80,7 @@ func TestQueryWithBefore(t *testing.T) { } //func TestQueryWithTagOnEntries(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "")}}) // require.Len(t, rs, 3) // assert.Equal(t, 31, rs[0].Date().Day()) // assert.Equal(t, 1, rs[1].Date().Day()) @@ -89,7 +89,7 @@ func TestQueryWithBefore(t *testing.T) { //} // //func TestQueryWithTagOnOverallSummary(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "")}}) // require.Len(t, rs, 4) // assert.Equal(t, 30, rs[0].Date().Day()) // assert.Equal(t, 1, rs[1].Date().Day()) @@ -99,7 +99,7 @@ func TestQueryWithBefore(t *testing.T) { //} // //func TestQueryWithTagOnEntriesAndInSummary(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", ""), klog.NewTagOrPanic("bar", "")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", ""), klog.NewTagOrPanic("bar", "")}}) // require.Len(t, rs, 2) // assert.Equal(t, 1, rs[0].Date().Day()) // assert.Equal(t, 3, rs[1].Date().Day()) @@ -107,48 +107,48 @@ func TestQueryWithBefore(t *testing.T) { //} // //func TestQueryWithTagValues(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "a")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "a")}}) // require.Len(t, rs, 1) // assert.Equal(t, 3, rs[0].Date().Day()) // assert.Equal(t, klog.NewDuration(8, 0), Total(rs...)) //} // //func TestQueryWithTagValuesInEntries(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "1")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "1")}}) // require.Len(t, rs, 1) // assert.Equal(t, 3, rs[0].Date().Day()) // assert.Equal(t, klog.NewDuration(4, 0), Total(rs...)) //} // //func TestQueryWithTagNonMatchingValues(t *testing.T) { -// rs := Query(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "3")}}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "3")}}) // require.Len(t, rs, 0) //} // //func TestQueryWithEntryTypes(t *testing.T) { // { -// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_DURATION}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_DURATION}) // require.Len(t, rs, 4) // assert.Equal(t, klog.NewDuration(0, 1545), Total(rs...)) // } // { -// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_NEGATIVE_DURATION}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_NEGATIVE_DURATION}) // require.Len(t, rs, 1) // assert.Equal(t, 1, rs[0].Date().Day()) // assert.Equal(t, klog.NewDuration(0, -30), Total(rs...)) // } // { -// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_POSITIVE_DURATION}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_POSITIVE_DURATION}) // require.Len(t, rs, 4) // assert.Equal(t, klog.NewDuration(0, 1575), Total(rs...)) // } // { -// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_RANGE}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_RANGE}) // require.Len(t, rs, 0) // assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) // } // { -// rs := Query(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_OPEN_RANGE}) +// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_OPEN_RANGE}) // require.Len(t, rs, 1) // assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) // } diff --git a/klog/service/kql/parse.go b/klog/service/kfl/parse.go similarity index 74% rename from klog/service/kql/parse.go rename to klog/service/kfl/parse.go index 9748ca9..6241be6 100644 --- a/klog/service/kql/parse.go +++ b/klog/service/kfl/parse.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "errors" @@ -9,17 +9,17 @@ import ( ) var ( - ErrMalformedQuery = errors.New("Malformed query") // This is only a just-in-case fallback. - ErrCannotMixAndOr = errors.New("Cannot mix && and || operators on the same level. Please use parenthesis () for grouping.") - ErrUnbalancedBrackets = errors.New("Unbalanced parenthesis. Please make sure that the number of opening and closing parentheses matches.") - errOperatorOperand = errors.New("Missing expected") // Internal “base” class - ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) - ErrOperandExpected = fmt.Errorf("%w operand. Please remove redundant logical operators.", errOperatorOperand) - ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") + ErrMalformedFilterQuery = errors.New("Malformed filter query") // This is only a just-in-case fallback. + ErrCannotMixAndOr = errors.New("Cannot mix && and || operators on the same level. Please use parenthesis () for grouping.") + ErrUnbalancedBrackets = errors.New("Unbalanced parenthesis. Please make sure that the number of opening and closing parentheses matches.") + errOperatorOperand = errors.New("Missing expected") // Internal “base” class + ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) + ErrOperandExpected = fmt.Errorf("%w operand. Please remove redundant logical operators.", errOperatorOperand) + ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") ) -func Parse(query string) (Predicate, error) { - tokens, err := tokenise(query) +func Parse(filterQuery string) (Predicate, error) { + tokens, err := tokenise(filterQuery) if err != nil { return nil, err } diff --git a/klog/service/kql/parse_test.go b/klog/service/kfl/parse_test.go similarity index 99% rename from klog/service/kql/parse_test.go rename to klog/service/kfl/parse_test.go index f169540..f858309 100644 --- a/klog/service/kql/parse_test.go +++ b/klog/service/kfl/parse_test.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "testing" diff --git a/klog/service/kql/predicate.go b/klog/service/kfl/predicate.go similarity index 99% rename from klog/service/kql/predicate.go rename to klog/service/kfl/predicate.go index 2de1347..1fa3693 100644 --- a/klog/service/kql/predicate.go +++ b/klog/service/kfl/predicate.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "errors" diff --git a/klog/service/kql/tokenise.go b/klog/service/kfl/tokenise.go similarity index 96% rename from klog/service/kql/tokenise.go rename to klog/service/kfl/tokenise.go index 0bc4c80..020d279 100644 --- a/klog/service/kql/tokenise.go +++ b/klog/service/kfl/tokenise.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "errors" @@ -43,8 +43,8 @@ var ( ErrUnrecognisedToken = errors.New("Unrecognised query token. Please make sure to use valid query syntax.") ) -func tokenise(query string) ([]token, error) { - txtParser := newTextParser(query) +func tokenise(filterQuery string) ([]token, error) { + txtParser := newTextParser(filterQuery) tokens := []token{} for { if txtParser.isFinished() { diff --git a/klog/service/kql/tokenise_test.go b/klog/service/kfl/tokenise_test.go similarity index 99% rename from klog/service/kql/tokenise_test.go rename to klog/service/kfl/tokenise_test.go index 374e172..b9b48d2 100644 --- a/klog/service/kql/tokenise_test.go +++ b/klog/service/kfl/tokenise_test.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "testing" diff --git a/klog/service/kql/util.go b/klog/service/kfl/util.go similarity index 97% rename from klog/service/kql/util.go rename to klog/service/kfl/util.go index b1c0422..4354b70 100644 --- a/klog/service/kql/util.go +++ b/klog/service/kfl/util.go @@ -1,4 +1,4 @@ -package kql +package kfl import ( "regexp" @@ -131,6 +131,6 @@ func (g *predicateGroup) make() (Predicate, error) { } else if g.operator == (tokenOr{}) { return Or{g.ps}, nil } else { - return nil, ErrMalformedQuery + return nil, ErrMalformedFilterQuery } } From 00f8635d2bddc4aead03f2ab3937a82f0877ff33 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Sun, 28 Dec 2025 10:14:26 +0100 Subject: [PATCH 05/10] Fix remaining tests --- klog/service/kfl/filter_test.go | 147 ++++++++++++++++---------------- 1 file changed, 73 insertions(+), 74 deletions(-) diff --git a/klog/service/kfl/filter_test.go b/klog/service/kfl/filter_test.go index 797b399..a79f581 100644 --- a/klog/service/kfl/filter_test.go +++ b/klog/service/kfl/filter_test.go @@ -79,77 +79,76 @@ func TestQueryWithBefore(t *testing.T) { assert.Equal(t, 1, rs[1].Date().Day()) } -//func TestQueryWithTagOnEntries(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "")}}) -// require.Len(t, rs, 3) -// assert.Equal(t, 31, rs[0].Date().Day()) -// assert.Equal(t, 1, rs[1].Date().Day()) -// assert.Equal(t, 3, rs[2].Date().Day()) -// assert.Equal(t, klog.NewDuration(5+8+6, 0), Total(rs...)) -//} -// -//func TestQueryWithTagOnOverallSummary(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "")}}) -// require.Len(t, rs, 4) -// assert.Equal(t, 30, rs[0].Date().Day()) -// assert.Equal(t, 1, rs[1].Date().Day()) -// assert.Equal(t, 2, rs[2].Date().Day()) -// assert.Equal(t, 3, rs[3].Date().Day()) -// assert.Equal(t, klog.NewDuration(6+7+8, -30+15), Total(rs...)) -//} -// -//func TestQueryWithTagOnEntriesAndInSummary(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", ""), klog.NewTagOrPanic("bar", "")}}) -// require.Len(t, rs, 2) -// assert.Equal(t, 1, rs[0].Date().Day()) -// assert.Equal(t, 3, rs[1].Date().Day()) -// assert.Equal(t, klog.NewDuration(8+6, 0), Total(rs...)) -//} -// -//func TestQueryWithTagValues(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("foo", "a")}}) -// require.Len(t, rs, 1) -// assert.Equal(t, 3, rs[0].Date().Day()) -// assert.Equal(t, klog.NewDuration(8, 0), Total(rs...)) -//} -// -//func TestQueryWithTagValuesInEntries(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "1")}}) -// require.Len(t, rs, 1) -// assert.Equal(t, 3, rs[0].Date().Day()) -// assert.Equal(t, klog.NewDuration(4, 0), Total(rs...)) -//} -// -//func TestQueryWithTagNonMatchingValues(t *testing.T) { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{Tags: []klog.Tag{klog.NewTagOrPanic("bar", "3")}}) -// require.Len(t, rs, 0) -//} -// -//func TestQueryWithEntryTypes(t *testing.T) { -// { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_DURATION}) -// require.Len(t, rs, 4) -// assert.Equal(t, klog.NewDuration(0, 1545), Total(rs...)) -// } -// { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_NEGATIVE_DURATION}) -// require.Len(t, rs, 1) -// assert.Equal(t, 1, rs[0].Date().Day()) -// assert.Equal(t, klog.NewDuration(0, -30), Total(rs...)) -// } -// { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_POSITIVE_DURATION}) -// require.Len(t, rs, 4) -// assert.Equal(t, klog.NewDuration(0, 1575), Total(rs...)) -// } -// { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_RANGE}) -// require.Len(t, rs, 0) -// assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) -// } -// { -// rs := Filter(sampleRecordsForQuerying(), FilterQry{EntryType: ENTRY_TYPE_OPEN_RANGE}) -// require.Len(t, rs, 1) -// assert.Equal(t, klog.NewDuration(0, 0), Total(rs...)) -// } -//} +func TestQueryWithTagOnEntries(t *testing.T) { + rs := Filter(HasTag{klog.NewTagOrPanic("bar", "")}, sampleRecordsForQuerying()) + require.Len(t, rs, 3) + assert.Equal(t, 31, rs[0].Date().Day()) + assert.Equal(t, 1, rs[1].Date().Day()) + assert.Equal(t, 3, rs[2].Date().Day()) + assert.Equal(t, klog.NewDuration(5+8+6, 0), service.Total(rs...)) +} + +func TestQueryWithTagOnOverallSummary(t *testing.T) { + rs := Filter(HasTag{klog.NewTagOrPanic("foo", "")}, sampleRecordsForQuerying()) + require.Len(t, rs, 3) + assert.Equal(t, 1, rs[0].Date().Day()) + assert.Equal(t, 2, rs[1].Date().Day()) + assert.Equal(t, 3, rs[2].Date().Day()) + assert.Equal(t, klog.NewDuration(6+7+8, -30+15), service.Total(rs...)) +} + +func TestQueryWithTagOnEntriesAndInSummary(t *testing.T) { + rs := Filter(And{[]Predicate{HasTag{klog.NewTagOrPanic("foo", "")}, HasTag{klog.NewTagOrPanic("bar", "")}}}, sampleRecordsForQuerying()) + require.Len(t, rs, 2) + assert.Equal(t, 1, rs[0].Date().Day()) + assert.Equal(t, 3, rs[1].Date().Day()) + assert.Equal(t, klog.NewDuration(8+6, 0), service.Total(rs...)) +} + +func TestQueryWithTagValues(t *testing.T) { + rs := Filter(HasTag{klog.NewTagOrPanic("foo", "a")}, sampleRecordsForQuerying()) + require.Len(t, rs, 1) + assert.Equal(t, 3, rs[0].Date().Day()) + assert.Equal(t, klog.NewDuration(8, 0), service.Total(rs...)) +} + +func TestQueryWithTagValuesInEntries(t *testing.T) { + rs := Filter(HasTag{klog.NewTagOrPanic("bar", "1")}, sampleRecordsForQuerying()) + require.Len(t, rs, 1) + assert.Equal(t, 3, rs[0].Date().Day()) + assert.Equal(t, klog.NewDuration(4, 0), service.Total(rs...)) +} + +func TestQueryWithTagNonMatchingValues(t *testing.T) { + rs := Filter(HasTag{klog.NewTagOrPanic("bar", "3")}, sampleRecordsForQuerying()) + require.Len(t, rs, 0) +} + +func TestQueryWithEntryTypes(t *testing.T) { + { + rs := Filter(IsEntryType{ENTRY_TYPE_DURATION}, sampleRecordsForQuerying()) + require.Len(t, rs, 4) + assert.Equal(t, klog.NewDuration(0, 1545), service.Total(rs...)) + } + { + rs := Filter(IsEntryType{ENTRY_TYPE_NEGATIVE_DURATION}, sampleRecordsForQuerying()) + require.Len(t, rs, 1) + assert.Equal(t, 1, rs[0].Date().Day()) + assert.Equal(t, klog.NewDuration(0, -30), service.Total(rs...)) + } + { + rs := Filter(IsEntryType{ENTRY_TYPE_POSITIVE_DURATION}, sampleRecordsForQuerying()) + require.Len(t, rs, 4) + assert.Equal(t, klog.NewDuration(0, 1575), service.Total(rs...)) + } + { + rs := Filter(IsEntryType{ENTRY_TYPE_RANGE}, sampleRecordsForQuerying()) + require.Len(t, rs, 0) + assert.Equal(t, klog.NewDuration(0, 0), service.Total(rs...)) + } + { + rs := Filter(IsEntryType{ENTRY_TYPE_OPEN_RANGE}, sampleRecordsForQuerying()) + require.Len(t, rs, 1) + assert.Equal(t, klog.NewDuration(0, 0), service.Total(rs...)) + } +} From eb40747c0a8edb98168406398c1380ed803e1c6b Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Sun, 28 Dec 2025 17:15:56 +0100 Subject: [PATCH 06/10] WIP --- klog/service/kfl/error.go | 94 +++++++++++++++++++++ klog/service/kfl/parse.go | 131 +++++++++++++++++++----------- klog/service/kfl/parse_test.go | 6 +- klog/service/kfl/tokenise.go | 63 +++++++++++--- klog/service/kfl/tokenise_test.go | 16 ++-- klog/service/kfl/util.go | 41 +++++++--- 6 files changed, 266 insertions(+), 85 deletions(-) create mode 100644 klog/service/kfl/error.go diff --git a/klog/service/kfl/error.go b/klog/service/kfl/error.go new file mode 100644 index 0000000..fdfe081 --- /dev/null +++ b/klog/service/kfl/error.go @@ -0,0 +1,94 @@ +package kfl + +import ( + "fmt" + "math" + "strings" + "unicode/utf8" +) + +type ParseError interface { + error + Original() error +} + +type parseError struct { + err error + position int + length int + query string +} + +func (e parseError) Error() string { + errorLength := int(math.Max(float64(e.length), 1)) + relevantQueryFragment, newStart := fuzzySubstr(e.query, e.position, errorLength) + return fmt.Sprintf( + // TODO remove   once reflower fix has been rebased in. + "%s\n\n    %s\n    %s%s%s\n    (Char %d in query.)", + e.err, + relevantQueryFragment, + strings.Repeat("—", newStart), + strings.Repeat("^", errorLength), + strings.Repeat("—", len(relevantQueryFragment)-(newStart+errorLength)), + e.position, + ) +} + +func (e parseError) Original() error { + return e.err +} + +func fuzzySubstr(text string, start int, length int) (string, int) { + if start < 0 || length < 0 || start >= len(text) { + return "", 0 + } + + // Clamp the end position to the text length + end := start + length + if end > len(text) { + end = len(text) + } + + // Find fuzzy start: go back at least 10 chars, up to 20, stop at first space after 10 + fuzzyStart := start + charCount := 0 + + for fuzzyStart > 0 && charCount < 20 { + // Move back one rune + _, size := utf8.DecodeLastRuneInString(text[:fuzzyStart]) + if size == 0 { + break + } + fuzzyStart -= size + charCount++ + + // If we've gone at least 10 chars and hit a space, stop here + if charCount >= 10 && text[fuzzyStart] == ' ' { + break + } + } + + // Find fuzzy end: go forward at least 10 chars, up to 20, stop at first space after 10 + fuzzyEnd := end + charCount = 0 + + for fuzzyEnd < len(text) && charCount < 20 { + r, size := utf8.DecodeRuneInString(text[fuzzyEnd:]) + if r == utf8.RuneError && size == 1 { + break + } + + // If we've gone at least 10 chars and hit a space, stop here + if charCount >= 10 && r == ' ' { + break + } + + fuzzyEnd += size + charCount++ + } + + // Calculate the translated position (where 'start' is in the returned substring) + translatedPos := start - fuzzyStart + + return text[fuzzyStart:fuzzyEnd], translatedPos +} diff --git a/klog/service/kfl/parse.go b/klog/service/kfl/parse.go index 6241be6..adbfd23 100644 --- a/klog/service/kfl/parse.go +++ b/klog/service/kfl/parse.go @@ -18,29 +18,45 @@ var ( ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") ) -func Parse(filterQuery string) (Predicate, error) { - tokens, err := tokenise(filterQuery) - if err != nil { - return nil, err - } - tp := newTokenParser(append(tokens, tokenCloseBracket{})) - p, err := parseGroup(&tp) - if err != nil { - return nil, err - } - // Check whether there are tokens left, which would indicate - // unbalanced brackets. - if tp.next() != nil { - return nil, ErrUnbalancedBrackets +func Parse(filterQuery string) (Predicate, ParseError) { + p, pErr := func() (Predicate, ParseError) { + tokens, pos, pErr := tokenise(filterQuery) + if pErr != nil { + return nil, pErr + } + tp := newTokenParser( + append(tokens, tokenCloseBracket{}), + append(pos, len(filterQuery)), + ) + p, pErr := parseGroup(&tp) + if pErr != nil { + return nil, pErr + } + // Check whether there are tokens left, which would indicate + // unbalanced brackets. + if nextToken, _ := tp.next(); nextToken != nil { + return nil, parseError{ + err: ErrUnbalancedBrackets, + position: len(filterQuery) - 1, + query: filterQuery, + } + } + return p, nil + }() + if pErr != nil { + if pErr, ok := pErr.(parseError); ok { + pErr.query = filterQuery + return nil, pErr + } } return p, nil } -func parseGroup(tp *tokenParser) (Predicate, error) { +func parseGroup(tp *tokenParser) (Predicate, ParseError) { g := predicateGroup{} for { - nextToken := tp.next() + nextToken, position := tp.next() if nextToken == nil { break } @@ -48,35 +64,38 @@ func parseGroup(tp *tokenParser) (Predicate, error) { switch tk := nextToken.(type) { case tokenOpenBracket: - if err := tp.checkNextIsOperand(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperand(); pErr != nil { + return nil, pErr } - p, err := parseGroup(tp) - if err != nil { - return nil, err + p, pErr := parseGroup(tp) + if pErr != nil { + return nil, pErr } g.append(p) case tokenCloseBracket: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } - p, err := g.make() - return p, err + p, pErr := g.make() + return p, pErr case tokenDate: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } date, err := klog.NewDateFromString(tk.date) if err != nil { - return nil, err + return nil, parseError{ + err: err, + position: position, + } } g.append(IsInDateRange{date, date}) case tokenDateRange: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } dateBoundaries := []klog.Date{nil, nil} for i, v := range tk.bounds { @@ -85,54 +104,67 @@ func parseGroup(tp *tokenParser) (Predicate, error) { } date, err := klog.NewDateFromString(v) if err != nil { - return nil, err + return nil, parseError{ + err: err, + position: position, + } } dateBoundaries[i] = date } g.append(IsInDateRange{dateBoundaries[0], dateBoundaries[1]}) case tokenPeriod: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } prd, err := period.NewPeriodFromPatternString(tk.period) if err != nil { - return nil, err + return nil, parseError{ + err: err, + position: position, + length: len(tk.period), + } } g.append(IsInDateRange{prd.Since(), prd.Until()}) case tokenAnd, tokenOr: - if err := tp.checkNextIsOperand(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperand(); pErr != nil { + return nil, pErr } - err := g.setOperator(tk) - if err != nil { - return nil, err + pErr := g.setOperator(tk, position) + if pErr != nil { + return nil, pErr } case tokenNot: - if err := tp.checkNextIsOperand(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperand(); pErr != nil { + return nil, pErr } g.negateNextOperand() case tokenEntryType: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } et, err := NewEntryTypeFromString(tk.entryType) if err != nil { - return nil, err + return nil, parseError{ + err: err, + position: position, + } } g.append(IsEntryType{et}) case tokenTag: - if err := tp.checkNextIsOperatorOrEnd(); err != nil { - return nil, err + if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { + return nil, pErr } tag, err := klog.NewTagFromString(tk.tag) if err != nil { - return nil, err + return nil, parseError{ + err: err, + position: position, + } } g.append(HasTag{tag}) @@ -141,5 +173,8 @@ func parseGroup(tp *tokenParser) (Predicate, error) { } } - return nil, ErrUnbalancedBrackets + return nil, parseError{ + err: ErrUnbalancedBrackets, + position: 0, + } } diff --git a/klog/service/kfl/parse_test.go b/klog/service/kfl/parse_test.go index f858309..8113500 100644 --- a/klog/service/kfl/parse_test.go +++ b/klog/service/kfl/parse_test.go @@ -47,7 +47,7 @@ func TestCannotMixAndOrOnSameLevel(t *testing.T) { } { t.Run(tt, func(t *testing.T) { p, err := Parse(tt) - require.ErrorIs(t, err, ErrCannotMixAndOr) + require.ErrorIs(t, err.Original(), ErrCannotMixAndOr) require.Nil(t, p) }) } @@ -164,7 +164,7 @@ func TestBracketMismatch(t *testing.T) { } { t.Run(tt, func(t *testing.T) { p, err := Parse(tt) - require.ErrorIs(t, err, ErrUnbalancedBrackets) + require.ErrorIs(t, err.Original(), ErrUnbalancedBrackets) require.Nil(t, p) }) } @@ -209,7 +209,7 @@ func TestOperatorOperandSequence(t *testing.T) { } { t.Run(tt, func(t *testing.T) { p, err := Parse(tt) - require.ErrorIs(t, err, errOperatorOperand) + require.ErrorIs(t, err.Original(), errOperatorOperand) require.Nil(t, p) }) } diff --git a/klog/service/kfl/tokenise.go b/klog/service/kfl/tokenise.go index 020d279..fa46b7b 100644 --- a/klog/service/kfl/tokenise.go +++ b/klog/service/kfl/tokenise.go @@ -2,12 +2,11 @@ package kfl import ( "errors" - "fmt" "regexp" "strings" ) -type token interface{} +type token any type tokenOpenBracket struct{} type tokenCloseBracket struct{} @@ -43,9 +42,10 @@ var ( ErrUnrecognisedToken = errors.New("Unrecognised query token. Please make sure to use valid query syntax.") ) -func tokenise(filterQuery string) ([]token, error) { +func tokenise(filterQuery string) ([]token, []int, ParseError) { txtParser := newTextParser(filterQuery) tokens := []token{} + pos := []int{} for { if txtParser.isFinished() { break @@ -55,66 +55,103 @@ func tokenise(filterQuery string) ([]token, error) { txtParser.advance(1) } else if txtParser.peekString("(") { tokens = append(tokens, tokenOpenBracket{}) + pos = append(pos, txtParser.pointer) txtParser.advance(1) } else if txtParser.peekString(")") { tokens = append(tokens, tokenCloseBracket{}) + pos = append(pos, txtParser.pointer) txtParser.advance(1) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if txtParser.peekString("&&") { tokens = append(tokens, tokenAnd{}) + pos = append(pos, txtParser.pointer) txtParser.advance(2) if !txtParser.peekString(EOT, " ") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if txtParser.peekString("||") { tokens = append(tokens, tokenOr{}) + pos = append(pos, txtParser.pointer) txtParser.advance(2) if !txtParser.peekString(EOT, " ") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if txtParser.peekString("!") { tokens = append(tokens, tokenNot{}) + pos = append(pos, txtParser.pointer) txtParser.advance(1) } else if tm := txtParser.peekRegex(tagRegex); tm != nil { value := tm[1] tokens = append(tokens, tokenTag{value}) + pos = append(pos, txtParser.pointer) txtParser.advance(1 + len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if ym := txtParser.peekRegex(typeRegex); ym != nil { tokens = append(tokens, tokenEntryType{ym[1]}) + pos = append(pos, txtParser.pointer) txtParser.advance(5 + len(ym[1])) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if rm := txtParser.peekRegex(dateRangeRegex); rm != nil { value := rm[1] parts := strings.Split(value, "...") tokens = append(tokens, tokenDateRange{parts}) + pos = append(pos, txtParser.pointer) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if dm := txtParser.peekRegex(dateRegex); dm != nil { value := dm[1] tokens = append(tokens, tokenDate{value}) + pos = append(pos, txtParser.pointer) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else if pm := txtParser.peekRegex(periodRegex); pm != nil { value := pm[1] tokens = append(tokens, tokenPeriod{value}) + pos = append(pos, txtParser.pointer) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, ErrMissingWhiteSpace + return nil, nil, parseError{ + err: ErrMissingWhiteSpace, + position: txtParser.pointer, + } } } else { - return nil, fmt.Errorf("%w: %s", ErrUnrecognisedToken, txtParser.remainder()) + return nil, nil, parseError{ + err: ErrUnrecognisedToken, + position: txtParser.pointer, + } } } - return tokens, nil + return tokens, pos, nil } diff --git a/klog/service/kfl/tokenise_test.go b/klog/service/kfl/tokenise_test.go index b9b48d2..1113cfb 100644 --- a/klog/service/kfl/tokenise_test.go +++ b/klog/service/kfl/tokenise_test.go @@ -9,19 +9,19 @@ import ( func TestTokeniseEmptyToken(t *testing.T) { { // Empty - p, err := tokenise("") + p, _, err := tokenise("") require.Nil(t, err) assert.Equal(t, p, []token{}) } { // Blank - p, err := tokenise(" ") + p, _, err := tokenise(" ") require.Nil(t, err) assert.Equal(t, p, []token{}) } } func TestTokeniseAllTokens(t *testing.T) { - p, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4) && type:duration") + p, _, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ tokenDate{"2020-01-01"}, @@ -40,7 +40,7 @@ func TestTokeniseAllTokens(t *testing.T) { } func TestDisregardWhitespaceBetweenTokens(t *testing.T) { - p, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) && type:duration") + p, _, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ tokenDate{"2020-01-01"}, @@ -66,8 +66,8 @@ func TestFailsOnUnrecognisedToken(t *testing.T) { "2020-01-01 {2020-01-02}", } { t.Run(txt, func(t *testing.T) { - p, err := tokenise(txt) - require.ErrorIs(t, err, ErrUnrecognisedToken) + p, _, err := tokenise(txt) + require.ErrorIs(t, err.Original(), ErrUnrecognisedToken) assert.Nil(t, p) }) } @@ -116,8 +116,8 @@ func TestFailsOnMissingWhitespace(t *testing.T) { "2020-Q4!( 2020-01-01 )", } { t.Run(txt, func(t *testing.T) { - p, err := tokenise(txt) - require.ErrorIs(t, err, ErrMissingWhiteSpace) + p, _, err := tokenise(txt) + require.ErrorIs(t, err.Original(), ErrMissingWhiteSpace) assert.Nil(t, p) }) } diff --git a/klog/service/kfl/util.go b/klog/service/kfl/util.go index 4354b70..946ef45 100644 --- a/klog/service/kfl/util.go +++ b/klog/service/kfl/util.go @@ -54,37 +54,46 @@ func (t *textParser) remainder() string { type tokenParser struct { tokens []token + pos []int pointer int } -func newTokenParser(ts []token) tokenParser { +func newTokenParser(ts []token, pos []int) tokenParser { return tokenParser{ tokens: ts, + pos: pos, pointer: 0, } } -func (t *tokenParser) next() token { +func (t *tokenParser) next() (token, int) { if t.pointer >= len(t.tokens) { - return nil + return nil, -1 } next := t.tokens[t.pointer] + pos := t.pos[t.pointer] t.pointer += 1 - return next + return next, pos } -func (t *tokenParser) checkNextIsOperand() error { +func (t *tokenParser) checkNextIsOperand() ParseError { if t.pointer >= len(t.tokens) { - return ErrOperandExpected + return parseError{ + err: ErrOperandExpected, + position: t.pos[t.pointer], + } } switch t.tokens[t.pointer].(type) { case tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot, tokenEntryType: return nil } - return ErrOperandExpected + return parseError{ + err: ErrOperandExpected, + position: t.pos[t.pointer], + } } -func (t *tokenParser) checkNextIsOperatorOrEnd() error { +func (t *tokenParser) checkNextIsOperatorOrEnd() ParseError { if t.pointer >= len(t.tokens) { return nil } @@ -92,7 +101,10 @@ func (t *tokenParser) checkNextIsOperatorOrEnd() error { case tokenCloseBracket, tokenAnd, tokenOr: return nil } - return ErrOperatorExpected + return parseError{ + err: ErrOperatorExpected, + position: t.pos[t.pointer], + } } type predicateGroup struct { @@ -109,12 +121,15 @@ func (g *predicateGroup) append(p Predicate) { g.ps = append(g.ps, p) } -func (g *predicateGroup) setOperator(operatorT token) error { +func (g *predicateGroup) setOperator(operatorT token, position int) ParseError { if g.operator == nil { g.operator = operatorT } if g.operator != operatorT { - return ErrCannotMixAndOr + return parseError{ + err: ErrCannotMixAndOr, + position: position, + } } return nil } @@ -123,7 +138,7 @@ func (g *predicateGroup) negateNextOperand() { g.isNextNegated = true } -func (g *predicateGroup) make() (Predicate, error) { +func (g *predicateGroup) make() (Predicate, ParseError) { if len(g.ps) == 1 { return g.ps[0], nil } else if g.operator == (tokenAnd{}) { @@ -131,6 +146,6 @@ func (g *predicateGroup) make() (Predicate, error) { } else if g.operator == (tokenOr{}) { return Or{g.ps}, nil } else { - return nil, ErrMalformedFilterQuery + return nil, parseError{err: ErrMalformedFilterQuery, position: 0} } } From 7f48f2476f11199b1d0ebab585d3d5a1fa7c5335 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Sun, 28 Dec 2025 18:59:56 +0100 Subject: [PATCH 07/10] Extract method --- klog/service/kfl/error.go | 63 +++------------------------------------ lib/text/substr.go | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 59 deletions(-) create mode 100644 lib/text/substr.go diff --git a/klog/service/kfl/error.go b/klog/service/kfl/error.go index fdfe081..2af5ba9 100644 --- a/klog/service/kfl/error.go +++ b/klog/service/kfl/error.go @@ -4,7 +4,8 @@ import ( "fmt" "math" "strings" - "unicode/utf8" + + tf "github.com/jotaen/klog/lib/text" ) type ParseError interface { @@ -21,10 +22,9 @@ type parseError struct { func (e parseError) Error() string { errorLength := int(math.Max(float64(e.length), 1)) - relevantQueryFragment, newStart := fuzzySubstr(e.query, e.position, errorLength) + relevantQueryFragment, newStart := tf.TextSubstrWithContext(e.query, e.position, errorLength, 10, 20) return fmt.Sprintf( - // TODO remove   once reflower fix has been rebased in. - "%s\n\n    %s\n    %s%s%s\n    (Char %d in query.)", + "%s\n\n%s\n%s%s%s\n(Char %d in query.)", e.err, relevantQueryFragment, strings.Repeat("—", newStart), @@ -37,58 +37,3 @@ func (e parseError) Error() string { func (e parseError) Original() error { return e.err } - -func fuzzySubstr(text string, start int, length int) (string, int) { - if start < 0 || length < 0 || start >= len(text) { - return "", 0 - } - - // Clamp the end position to the text length - end := start + length - if end > len(text) { - end = len(text) - } - - // Find fuzzy start: go back at least 10 chars, up to 20, stop at first space after 10 - fuzzyStart := start - charCount := 0 - - for fuzzyStart > 0 && charCount < 20 { - // Move back one rune - _, size := utf8.DecodeLastRuneInString(text[:fuzzyStart]) - if size == 0 { - break - } - fuzzyStart -= size - charCount++ - - // If we've gone at least 10 chars and hit a space, stop here - if charCount >= 10 && text[fuzzyStart] == ' ' { - break - } - } - - // Find fuzzy end: go forward at least 10 chars, up to 20, stop at first space after 10 - fuzzyEnd := end - charCount = 0 - - for fuzzyEnd < len(text) && charCount < 20 { - r, size := utf8.DecodeRuneInString(text[fuzzyEnd:]) - if r == utf8.RuneError && size == 1 { - break - } - - // If we've gone at least 10 chars and hit a space, stop here - if charCount >= 10 && r == ' ' { - break - } - - fuzzyEnd += size - charCount++ - } - - // Calculate the translated position (where 'start' is in the returned substring) - translatedPos := start - fuzzyStart - - return text[fuzzyStart:fuzzyEnd], translatedPos -} diff --git a/lib/text/substr.go b/lib/text/substr.go new file mode 100644 index 0000000..78091be --- /dev/null +++ b/lib/text/substr.go @@ -0,0 +1,60 @@ +package text + +import "unicode/utf8" + +// TextSubstrWithContext returns a fragment of a string like a regular `substr` +// method would do. However, it returns a bit of surrounding text for context. +// The surrounding text is between `minSurroundingRunes` and `maxSurroundingRunes` +// long, and it tries to find a word boundary (space character) as natural cut-off. +// Only if it cannot find one, it makes a hard cut. +func TextSubstrWithContext(text string, start int, length int, minSurroundingRunes int, maxSurroundingRunes int) (string, int) { + if start < 0 || length < 0 || start >= len(text) { + return "", 0 + } + + end := start + length + if end > len(text) { + end = len(text) + } + + fuzzyStart := start + charCount := 0 + + for fuzzyStart > 0 && charCount < maxSurroundingRunes { + _, size := utf8.DecodeLastRuneInString(text[:fuzzyStart]) + if size == 0 { + break + } + fuzzyStart -= size + charCount++ + + if charCount >= minSurroundingRunes && text[fuzzyStart] == ' ' { + break + } + } + + if fuzzyStart < len(text) && text[fuzzyStart] == ' ' { + fuzzyStart++ + } + + fuzzyEnd := end + charCount = 0 + + for fuzzyEnd < len(text) && charCount < maxSurroundingRunes { + r, size := utf8.DecodeRuneInString(text[fuzzyEnd:]) + if r == utf8.RuneError && size == 1 { + break + } + + if charCount >= minSurroundingRunes && r == ' ' { + break + } + + fuzzyEnd += size + charCount++ + } + + translatedPos := start - fuzzyStart + + return text[fuzzyStart:fuzzyEnd], translatedPos +} From 44c09da0769da135d3105c60a1142e8247ea43d2 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Sun, 28 Dec 2025 19:07:14 +0100 Subject: [PATCH 08/10] Move --- klog/service/kfl/error.go | 2 +- lib/{text => terminalformat}/substr.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename lib/{text => terminalformat}/substr.go (98%) diff --git a/klog/service/kfl/error.go b/klog/service/kfl/error.go index 2af5ba9..e4b4673 100644 --- a/klog/service/kfl/error.go +++ b/klog/service/kfl/error.go @@ -5,7 +5,7 @@ import ( "math" "strings" - tf "github.com/jotaen/klog/lib/text" + tf "github.com/jotaen/klog/lib/terminalformat" ) type ParseError interface { diff --git a/lib/text/substr.go b/lib/terminalformat/substr.go similarity index 98% rename from lib/text/substr.go rename to lib/terminalformat/substr.go index 78091be..a28ff04 100644 --- a/lib/text/substr.go +++ b/lib/terminalformat/substr.go @@ -1,4 +1,4 @@ -package text +package terminalformat import "unicode/utf8" From 563f5392a4f606e7eaf686690090249839940219 Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Mon, 29 Dec 2025 01:13:16 +0100 Subject: [PATCH 09/10] Error lengths --- klog/service/kfl/error.go | 20 +++++++++++---- klog/service/kfl/parse.go | 41 +++++++++++++++++++------------ klog/service/kfl/parse_test.go | 45 +++++++++++++++++++++++++--------- klog/service/kfl/tokenise.go | 19 ++++++++++---- klog/service/kfl/util.go | 7 +++++- 5 files changed, 94 insertions(+), 38 deletions(-) diff --git a/klog/service/kfl/error.go b/klog/service/kfl/error.go index e4b4673..bbc931a 100644 --- a/klog/service/kfl/error.go +++ b/klog/service/kfl/error.go @@ -11,6 +11,7 @@ import ( type ParseError interface { error Original() error + Position() (int, int) } type parseError struct { @@ -21,19 +22,28 @@ type parseError struct { } func (e parseError) Error() string { - errorLength := int(math.Max(float64(e.length), 1)) + errorLength := max(e.length, 1) relevantQueryFragment, newStart := tf.TextSubstrWithContext(e.query, e.position, errorLength, 10, 20) return fmt.Sprintf( - "%s\n\n%s\n%s%s%s\n(Char %d in query.)", + "%s\n\n%s\n%s%s%s\nCursor positions %d-%d in query.", e.err, relevantQueryFragment, - strings.Repeat("—", newStart), - strings.Repeat("^", errorLength), - strings.Repeat("—", len(relevantQueryFragment)-(newStart+errorLength)), + strings.Repeat("—", max(0, newStart)), + strings.Repeat("^", max(0, errorLength)), + strings.Repeat("—", max(0, len(relevantQueryFragment)-(newStart+errorLength))), e.position, + e.position+errorLength, ) } func (e parseError) Original() error { return e.err } + +func (e parseError) Position() (int, int) { + return e.position, e.length +} + +func max(x int, y int) int { + return int(math.Max(float64(x), float64(y))) +} diff --git a/klog/service/kfl/parse.go b/klog/service/kfl/parse.go index adbfd23..405172b 100644 --- a/klog/service/kfl/parse.go +++ b/klog/service/kfl/parse.go @@ -3,19 +3,22 @@ package kfl import ( "errors" "fmt" + "strings" "github.com/jotaen/klog/klog" "github.com/jotaen/klog/klog/service/period" ) var ( - ErrMalformedFilterQuery = errors.New("Malformed filter query") // This is only a just-in-case fallback. - ErrCannotMixAndOr = errors.New("Cannot mix && and || operators on the same level. Please use parenthesis () for grouping.") - ErrUnbalancedBrackets = errors.New("Unbalanced parenthesis. Please make sure that the number of opening and closing parentheses matches.") - errOperatorOperand = errors.New("Missing expected") // Internal “base” class - ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) - ErrOperandExpected = fmt.Errorf("%w operand. Please remove redundant logical operators.", errOperatorOperand) - ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") + ErrMalformedFilterQuery = errors.New("Malformed filter query") // This is only a just-in-case fallback. + ErrCannotMixAndOr = errors.New("Cannot mix && and || operators on the same level. Please use parenthesis () for grouping.") + errUnbalancedBrackets = errors.New("Missing") // Internal “base” class + ErrUnbalancedOpenBracket = fmt.Errorf("%w opening parenthesis. Please make sure that the number of opening and closing parentheses matches.", errUnbalancedBrackets) + ErrUnbalancedCloseBracket = fmt.Errorf("%w closing parenthesis. Please make sure that the number of opening and closing parentheses matches.", errUnbalancedBrackets) + errOperatorOperand = errors.New("Missing") // Internal “base” class + ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) + ErrOperandExpected = fmt.Errorf("%w filter term. Please remove redundant logical operators.", errOperatorOperand) + ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") ) func Parse(filterQuery string) (Predicate, ParseError) { @@ -36,9 +39,9 @@ func Parse(filterQuery string) (Predicate, ParseError) { // unbalanced brackets. if nextToken, _ := tp.next(); nextToken != nil { return nil, parseError{ - err: ErrUnbalancedBrackets, - position: len(filterQuery) - 1, - query: filterQuery, + err: ErrUnbalancedOpenBracket, + position: 0, + length: len(filterQuery), } } return p, nil @@ -55,10 +58,17 @@ func Parse(filterQuery string) (Predicate, ParseError) { func parseGroup(tp *tokenParser) (Predicate, ParseError) { g := predicateGroup{} + if pErr := tp.checkNextIsOperand(); pErr != nil { + return nil, pErr + } + for { nextToken, position := tp.next() if nextToken == nil { - break + return nil, parseError{ + err: ErrUnbalancedCloseBracket, + position: 0, + } } switch tk := nextToken.(type) { @@ -89,6 +99,7 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { return nil, parseError{ err: err, position: position, + length: len(tk.date), } } g.append(IsInDateRange{date, date}) @@ -107,6 +118,7 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { return nil, parseError{ err: err, position: position, + length: len(strings.Join(tk.bounds, "...")), } } dateBoundaries[i] = date @@ -151,6 +163,7 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { return nil, parseError{ err: err, position: position, + length: len("type:") + len(tk.entryType), } } g.append(IsEntryType{et}) @@ -164,6 +177,7 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { return nil, parseError{ err: err, position: position, + length: len(tk.tag), } } g.append(HasTag{tag}) @@ -172,9 +186,4 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { panic("Unrecognized token") } } - - return nil, parseError{ - err: ErrUnbalancedBrackets, - position: 0, - } } diff --git a/klog/service/kfl/parse_test.go b/klog/service/kfl/parse_test.go index 8113500..abe3114 100644 --- a/klog/service/kfl/parse_test.go +++ b/klog/service/kfl/parse_test.go @@ -8,6 +8,12 @@ import ( "github.com/stretchr/testify/require" ) +type et struct { // “Error Test” + input string + pos int + len int +} + func TestAtDate(t *testing.T) { p, err := Parse("2020-03-01") require.Nil(t, err) @@ -153,19 +159,36 @@ func TestTags(t *testing.T) { }}, p) } -func TestBracketMismatch(t *testing.T) { - for _, tt := range []string{ - "(2020-01", - "((2020-01", - "(2020-01-01))", - "2020-01-01)", - "(2020-01-01 && (2020-02-02 || 2020-03-03", - "(2020-01-01 && (2020-02-02))) || 2020-03-03", +func TestOpeningBracketMismatch(t *testing.T) { + for _, tt := range []et{ + {"(2020-01", 0, 0}, + {"((2020-01", 0, 0}, + {"(2020-01-01 && (2020-02-02 || 2020-03-03", 0, 0}, } { - t.Run(tt, func(t *testing.T) { - p, err := Parse(tt) - require.ErrorIs(t, err.Original(), ErrUnbalancedBrackets) + t.Run(tt.input, func(t *testing.T) { + p, err := Parse(tt.input) + require.Nil(t, p) + require.ErrorIs(t, err.Original(), errUnbalancedBrackets) + pos, len := err.Position() + assert.Equal(t, tt.pos, pos) + assert.Equal(t, tt.len, len) + }) + } +} + +func TestClosingBracketMismatch(t *testing.T) { + for _, tt := range []et{ + {"(2020-01-01))", 0, 13}, + {"2020-01-01)", 0, 11}, + {"(2020-01-01 && (2020-02-02))) || 2020-03-03", 0, 43}, + } { + t.Run(tt.input, func(t *testing.T) { + p, err := Parse(tt.input) require.Nil(t, p) + require.ErrorIs(t, err.Original(), errUnbalancedBrackets) + pos, len := err.Position() + assert.Equal(t, tt.pos, pos) + assert.Equal(t, tt.len, len) }) } } diff --git a/klog/service/kfl/tokenise.go b/klog/service/kfl/tokenise.go index fa46b7b..2863657 100644 --- a/klog/service/kfl/tokenise.go +++ b/klog/service/kfl/tokenise.go @@ -64,7 +64,8 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if !txtParser.peekString(EOT, " ", ")") { return nil, nil, parseError{ err: ErrMissingWhiteSpace, - position: txtParser.pointer, + position: txtParser.pointer - 1, + length: 1, } } } else if txtParser.peekString("&&") { @@ -75,6 +76,7 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { return nil, nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, + length: 1, } } } else if txtParser.peekString("||") { @@ -85,6 +87,7 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { return nil, nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, + length: 1, } } } else if txtParser.peekString("!") { @@ -100,6 +103,7 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { return nil, nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, + length: 1, } } } else if ym := txtParser.peekRegex(typeRegex); ym != nil { @@ -109,7 +113,8 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if !txtParser.peekString(EOT, " ", ")") { return nil, nil, parseError{ err: ErrMissingWhiteSpace, - position: txtParser.pointer, + position: txtParser.pointer - 1, + length: 1, } } } else if rm := txtParser.peekRegex(dateRangeRegex); rm != nil { @@ -121,7 +126,8 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if !txtParser.peekString(EOT, " ", ")") { return nil, nil, parseError{ err: ErrMissingWhiteSpace, - position: txtParser.pointer, + position: txtParser.pointer - 1, + length: 1, } } } else if dm := txtParser.peekRegex(dateRegex); dm != nil { @@ -132,7 +138,8 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if !txtParser.peekString(EOT, " ", ")") { return nil, nil, parseError{ err: ErrMissingWhiteSpace, - position: txtParser.pointer, + position: txtParser.pointer - 1, + length: 1, } } } else if pm := txtParser.peekRegex(periodRegex); pm != nil { @@ -143,13 +150,15 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if !txtParser.peekString(EOT, " ", ")") { return nil, nil, parseError{ err: ErrMissingWhiteSpace, - position: txtParser.pointer, + position: txtParser.pointer - 1, + length: 1, } } } else { return nil, nil, parseError{ err: ErrUnrecognisedToken, position: txtParser.pointer, + length: 1, } } } diff --git a/klog/service/kfl/util.go b/klog/service/kfl/util.go index 946ef45..aef23ba 100644 --- a/klog/service/kfl/util.go +++ b/klog/service/kfl/util.go @@ -129,6 +129,7 @@ func (g *predicateGroup) setOperator(operatorT token, position int) ParseError { return parseError{ err: ErrCannotMixAndOr, position: position, + length: 2, } } return nil @@ -146,6 +147,10 @@ func (g *predicateGroup) make() (Predicate, ParseError) { } else if g.operator == (tokenOr{}) { return Or{g.ps}, nil } else { - return nil, parseError{err: ErrMalformedFilterQuery, position: 0} + // This would happen for an empty group. + return nil, parseError{ + err: ErrMalformedFilterQuery, + position: 0, + } } } From 8e2bac01c71594572bd5fdec30b1adee35c6276f Mon Sep 17 00:00:00 2001 From: Jan Heuermann Date: Mon, 29 Dec 2025 12:56:18 +0100 Subject: [PATCH 10/10] Simplify token type(s) --- klog/service/kfl/parse.go | 50 +++++++-------- klog/service/kfl/tokenise.go | 101 +++++++++++++----------------- klog/service/kfl/tokenise_test.go | 60 +++++++++--------- klog/service/kfl/util.go | 58 ++++++++++------- 4 files changed, 134 insertions(+), 135 deletions(-) diff --git a/klog/service/kfl/parse.go b/klog/service/kfl/parse.go index 405172b..3040c32 100644 --- a/klog/service/kfl/parse.go +++ b/klog/service/kfl/parse.go @@ -16,20 +16,19 @@ var ( ErrUnbalancedOpenBracket = fmt.Errorf("%w opening parenthesis. Please make sure that the number of opening and closing parentheses matches.", errUnbalancedBrackets) ErrUnbalancedCloseBracket = fmt.Errorf("%w closing parenthesis. Please make sure that the number of opening and closing parentheses matches.", errUnbalancedBrackets) errOperatorOperand = errors.New("Missing") // Internal “base” class - ErrOperatorExpected = fmt.Errorf("%w operator. Please put logical operators ('&&' or '||') between the search operands.", errOperatorOperand) + ErrOperatorExpected = fmt.Errorf("%w operator. Please put a logical operator ('&&' or '||') before this search operand.", errOperatorOperand) ErrOperandExpected = fmt.Errorf("%w filter term. Please remove redundant logical operators.", errOperatorOperand) ErrIllegalTokenValue = errors.New("Illegal value. Please make sure to use only valid operand values.") ) func Parse(filterQuery string) (Predicate, ParseError) { p, pErr := func() (Predicate, ParseError) { - tokens, pos, pErr := tokenise(filterQuery) + tokens, pErr := tokenise(filterQuery) if pErr != nil { return nil, pErr } tp := newTokenParser( - append(tokens, tokenCloseBracket{}), - append(pos, len(filterQuery)), + append(tokens, token{tokenCloseBracket, ")", len(filterQuery) - 1}), ) p, pErr := parseGroup(&tp) if pErr != nil { @@ -37,7 +36,7 @@ func Parse(filterQuery string) (Predicate, ParseError) { } // Check whether there are tokens left, which would indicate // unbalanced brackets. - if nextToken, _ := tp.next(); nextToken != nil { + if tp.next() != (token{}) { return nil, parseError{ err: ErrUnbalancedOpenBracket, position: 0, @@ -56,22 +55,22 @@ func Parse(filterQuery string) (Predicate, ParseError) { } func parseGroup(tp *tokenParser) (Predicate, ParseError) { - g := predicateGroup{} + g := newPredicateGroup() if pErr := tp.checkNextIsOperand(); pErr != nil { return nil, pErr } for { - nextToken, position := tp.next() - if nextToken == nil { + tk := tp.next() + if tk == (token{}) { return nil, parseError{ err: ErrUnbalancedCloseBracket, position: 0, } } - switch tk := nextToken.(type) { + switch tk.kind { case tokenOpenBracket: if pErr := tp.checkNextIsOperand(); pErr != nil { @@ -94,12 +93,12 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { return nil, pErr } - date, err := klog.NewDateFromString(tk.date) + date, err := klog.NewDateFromString(tk.value) if err != nil { return nil, parseError{ err: err, - position: position, - length: len(tk.date), + position: tk.position, + length: len(tk.value), } } g.append(IsInDateRange{date, date}) @@ -109,7 +108,8 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { return nil, pErr } dateBoundaries := []klog.Date{nil, nil} - for i, v := range tk.bounds { + bounds := strings.Split(tk.value, "...") + for i, v := range bounds { if v == "" { continue } @@ -117,8 +117,8 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if err != nil { return nil, parseError{ err: err, - position: position, - length: len(strings.Join(tk.bounds, "...")), + position: tk.position, + length: len(tk.value), } } dateBoundaries[i] = date @@ -129,12 +129,12 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { return nil, pErr } - prd, err := period.NewPeriodFromPatternString(tk.period) + prd, err := period.NewPeriodFromPatternString(tk.value) if err != nil { return nil, parseError{ err: err, - position: position, - length: len(tk.period), + position: tk.position, + length: len(tk.value), } } g.append(IsInDateRange{prd.Since(), prd.Until()}) @@ -143,7 +143,7 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if pErr := tp.checkNextIsOperand(); pErr != nil { return nil, pErr } - pErr := g.setOperator(tk, position) + pErr := g.setOperator(tk, tk.position) if pErr != nil { return nil, pErr } @@ -158,12 +158,12 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { return nil, pErr } - et, err := NewEntryTypeFromString(tk.entryType) + et, err := NewEntryTypeFromString(strings.TrimLeft(tk.value, "type:")) if err != nil { return nil, parseError{ err: err, - position: position, - length: len("type:") + len(tk.entryType), + position: tk.position, + length: len(tk.value), } } g.append(IsEntryType{et}) @@ -172,12 +172,12 @@ func parseGroup(tp *tokenParser) (Predicate, ParseError) { if pErr := tp.checkNextIsOperatorOrEnd(); pErr != nil { return nil, pErr } - tag, err := klog.NewTagFromString(tk.tag) + tag, err := klog.NewTagFromString(tk.value) if err != nil { return nil, parseError{ err: err, - position: position, - length: len(tk.tag), + position: tk.position, + length: len(tk.value), } } g.append(HasTag{tag}) diff --git a/klog/service/kfl/tokenise.go b/klog/service/kfl/tokenise.go index 2863657..f02e8bc 100644 --- a/klog/service/kfl/tokenise.go +++ b/klog/service/kfl/tokenise.go @@ -3,38 +3,35 @@ package kfl import ( "errors" "regexp" - "strings" ) -type token any +type tokenKind int -type tokenOpenBracket struct{} -type tokenCloseBracket struct{} -type tokenAnd struct{} -type tokenOr struct{} -type tokenNot struct{} -type tokenDate struct { - date string -} -type tokenPeriod struct { - period string -} -type tokenDateRange struct { - bounds []string -} -type tokenTag struct { - tag string -} -type tokenEntryType struct { - entryType string +const ( + tokenOpenBracket tokenKind = iota + tokenCloseBracket + tokenAnd + tokenOr + tokenNot + tokenDate + tokenPeriod + tokenDateRange + tokenTag + tokenEntryType +) + +type token struct { + kind tokenKind + value string + position int } var ( - tagRegex = regexp.MustCompile(`^#(([\p{L}\d_-]+)(=(("[^"]*")|('[^']*')|([\p{L}\d_-]*)))?)`) + tagRegex = regexp.MustCompile(`^(#([\p{L}\d_-]+)(=(("[^"]*")|('[^']*')|([\p{L}\d_-]*)))?)`) dateRangeRegex = regexp.MustCompile(`^((\d{4}-\d{2}-\d{2})?\.\.\.(\d{4}-\d{2}-\d{2})?)`) dateRegex = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2})`) periodRegex = regexp.MustCompile(`^((\d{4}-\p{L}?\d+)|(\d{4}))`) - typeRegex = regexp.MustCompile(`^type:([\p{L}\-_]+)`) + typeRegex = regexp.MustCompile(`^(type:[\p{L}\-_]+)`) ) var ( @@ -42,10 +39,9 @@ var ( ErrUnrecognisedToken = errors.New("Unrecognised query token. Please make sure to use valid query syntax.") ) -func tokenise(filterQuery string) ([]token, []int, ParseError) { +func tokenise(filterQuery string) ([]token, ParseError) { txtParser := newTextParser(filterQuery) tokens := []token{} - pos := []int{} for { if txtParser.isFinished() { break @@ -54,64 +50,57 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { if txtParser.peekString(" ") { txtParser.advance(1) } else if txtParser.peekString("(") { - tokens = append(tokens, tokenOpenBracket{}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenOpenBracket, "(", txtParser.pointer}) txtParser.advance(1) } else if txtParser.peekString(")") { - tokens = append(tokens, tokenCloseBracket{}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenCloseBracket, ")", txtParser.pointer}) txtParser.advance(1) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer - 1, length: 1, } } } else if txtParser.peekString("&&") { - tokens = append(tokens, tokenAnd{}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenAnd, "&&", txtParser.pointer}) txtParser.advance(2) if !txtParser.peekString(EOT, " ") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, length: 1, } } } else if txtParser.peekString("||") { - tokens = append(tokens, tokenOr{}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenOr, "||", txtParser.pointer}) txtParser.advance(2) if !txtParser.peekString(EOT, " ") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, length: 1, } } } else if txtParser.peekString("!") { - tokens = append(tokens, tokenNot{}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenNot, "!", txtParser.pointer}) txtParser.advance(1) } else if tm := txtParser.peekRegex(tagRegex); tm != nil { value := tm[1] - tokens = append(tokens, tokenTag{value}) - pos = append(pos, txtParser.pointer) - txtParser.advance(1 + len(value)) + tokens = append(tokens, token{tokenTag, value, txtParser.pointer}) + txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer, length: 1, } } } else if ym := txtParser.peekRegex(typeRegex); ym != nil { - tokens = append(tokens, tokenEntryType{ym[1]}) - pos = append(pos, txtParser.pointer) - txtParser.advance(5 + len(ym[1])) + tokens = append(tokens, token{tokenEntryType, ym[1], txtParser.pointer}) + txtParser.advance(len(ym[1])) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer - 1, length: 1, @@ -119,12 +108,10 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { } } else if rm := txtParser.peekRegex(dateRangeRegex); rm != nil { value := rm[1] - parts := strings.Split(value, "...") - tokens = append(tokens, tokenDateRange{parts}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenDateRange, value, txtParser.pointer}) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer - 1, length: 1, @@ -132,11 +119,10 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { } } else if dm := txtParser.peekRegex(dateRegex); dm != nil { value := dm[1] - tokens = append(tokens, tokenDate{value}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenDate, value, txtParser.pointer}) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer - 1, length: 1, @@ -144,23 +130,22 @@ func tokenise(filterQuery string) ([]token, []int, ParseError) { } } else if pm := txtParser.peekRegex(periodRegex); pm != nil { value := pm[1] - tokens = append(tokens, tokenPeriod{value}) - pos = append(pos, txtParser.pointer) + tokens = append(tokens, token{tokenPeriod, value, txtParser.pointer}) txtParser.advance(len(value)) if !txtParser.peekString(EOT, " ", ")") { - return nil, nil, parseError{ + return nil, parseError{ err: ErrMissingWhiteSpace, position: txtParser.pointer - 1, length: 1, } } } else { - return nil, nil, parseError{ + return nil, parseError{ err: ErrUnrecognisedToken, position: txtParser.pointer, length: 1, } } } - return tokens, pos, nil + return tokens, nil } diff --git a/klog/service/kfl/tokenise_test.go b/klog/service/kfl/tokenise_test.go index 1113cfb..6280257 100644 --- a/klog/service/kfl/tokenise_test.go +++ b/klog/service/kfl/tokenise_test.go @@ -9,52 +9,52 @@ import ( func TestTokeniseEmptyToken(t *testing.T) { { // Empty - p, _, err := tokenise("") + p, err := tokenise("") require.Nil(t, err) assert.Equal(t, p, []token{}) } { // Blank - p, _, err := tokenise(" ") + p, err := tokenise(" ") require.Nil(t, err) assert.Equal(t, p, []token{}) } } func TestTokeniseAllTokens(t *testing.T) { - p, _, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4) && type:duration") + p, err := tokenise("2020-01-01 && #hello || (2020-02-02 && !2021-Q4) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ - tokenDate{"2020-01-01"}, - tokenAnd{}, - tokenTag{"hello"}, - tokenOr{}, - tokenOpenBracket{}, - tokenDate{"2020-02-02"}, - tokenAnd{}, - tokenNot{}, - tokenPeriod{"2021-Q4"}, - tokenCloseBracket{}, - tokenAnd{}, - tokenEntryType{"duration"}, + {tokenDate, "2020-01-01", 0}, + {tokenAnd, "&&", 11}, + {tokenTag, "#hello", 14}, + {tokenOr, "||", 21}, + {tokenOpenBracket, "(", 24}, + {tokenDate, "2020-02-02", 25}, + {tokenAnd, "&&", 36}, + {tokenNot, "!", 39}, + {tokenPeriod, "2021-Q4", 40}, + {tokenCloseBracket, ")", 47}, + {tokenAnd, "&&", 49}, + {tokenEntryType, "type:duration", 52}, }, p) } func TestDisregardWhitespaceBetweenTokens(t *testing.T) { - p, _, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) && type:duration") + p, err := tokenise(" 2020-01-01 && #hello || ( 2020-02-02 && ! 2021-Q4 ) && type:duration") require.Nil(t, err) assert.Equal(t, []token{ - tokenDate{"2020-01-01"}, - tokenAnd{}, - tokenTag{"hello"}, - tokenOr{}, - tokenOpenBracket{}, - tokenDate{"2020-02-02"}, - tokenAnd{}, - tokenNot{}, - tokenPeriod{"2021-Q4"}, - tokenCloseBracket{}, - tokenAnd{}, - tokenEntryType{"duration"}, + {tokenDate, "2020-01-01", 3}, + {tokenAnd, "&&", 17}, + {tokenTag, "#hello", 24}, + {tokenOr, "||", 34}, + {tokenOpenBracket, "(", 40}, + {tokenDate, "2020-02-02", 44}, + {tokenAnd, "&&", 57}, + {tokenNot, "!", 62}, + {tokenPeriod, "2021-Q4", 66}, + {tokenCloseBracket, ")", 75}, + {tokenAnd, "&&", 78}, + {tokenEntryType, "type:duration", 84}, }, p) } @@ -66,7 +66,7 @@ func TestFailsOnUnrecognisedToken(t *testing.T) { "2020-01-01 {2020-01-02}", } { t.Run(txt, func(t *testing.T) { - p, _, err := tokenise(txt) + p, err := tokenise(txt) require.ErrorIs(t, err.Original(), ErrUnrecognisedToken) assert.Nil(t, p) }) @@ -116,7 +116,7 @@ func TestFailsOnMissingWhitespace(t *testing.T) { "2020-Q4!( 2020-01-01 )", } { t.Run(txt, func(t *testing.T) { - p, _, err := tokenise(txt) + p, err := tokenise(txt) require.ErrorIs(t, err.Original(), ErrMissingWhiteSpace) assert.Nil(t, p) }) diff --git a/klog/service/kfl/util.go b/klog/service/kfl/util.go index aef23ba..d357ff5 100644 --- a/klog/service/kfl/util.go +++ b/klog/service/kfl/util.go @@ -54,42 +54,44 @@ func (t *textParser) remainder() string { type tokenParser struct { tokens []token - pos []int pointer int } -func newTokenParser(ts []token, pos []int) tokenParser { +func newTokenParser(ts []token) tokenParser { return tokenParser{ tokens: ts, - pos: pos, pointer: 0, } } -func (t *tokenParser) next() (token, int) { +func (t *tokenParser) next() token { if t.pointer >= len(t.tokens) { - return nil, -1 + return token{} } next := t.tokens[t.pointer] - pos := t.pos[t.pointer] t.pointer += 1 - return next, pos + return next } func (t *tokenParser) checkNextIsOperand() ParseError { if t.pointer >= len(t.tokens) { return parseError{ err: ErrOperandExpected, - position: t.pos[t.pointer], + position: t.tokens[len(t.tokens)-1].position, + length: 1, } } - switch t.tokens[t.pointer].(type) { - case tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot, tokenEntryType: - return nil + for _, k := range []tokenKind{ + tokenOpenBracket, tokenTag, tokenDate, tokenDateRange, tokenPeriod, tokenNot, tokenEntryType, + } { + if t.tokens[t.pointer].kind == k { + return nil + } } return parseError{ err: ErrOperandExpected, - position: t.pos[t.pointer], + position: t.tokens[t.pointer].position, + length: len(t.tokens[t.pointer].value), } } @@ -97,22 +99,34 @@ func (t *tokenParser) checkNextIsOperatorOrEnd() ParseError { if t.pointer >= len(t.tokens) { return nil } - switch t.tokens[t.pointer].(type) { - case tokenCloseBracket, tokenAnd, tokenOr: - return nil + for _, k := range []tokenKind{ + tokenCloseBracket, tokenAnd, tokenOr, + } { + if t.tokens[t.pointer].kind == k { + return nil + } } return parseError{ err: ErrOperatorExpected, - position: t.pos[t.pointer], + position: t.tokens[t.pointer].position, + length: len(t.tokens[t.pointer].value), } } type predicateGroup struct { ps []Predicate - operator token // nil or tokenAnd or tokenOr + operator tokenKind // -1 (unset) or tokenAnd or tokenOr isNextNegated bool } +func newPredicateGroup() predicateGroup { + return predicateGroup{ + ps: nil, + operator: -1, + isNextNegated: false, + } +} + func (g *predicateGroup) append(p Predicate) { if g.isNextNegated { g.isNextNegated = false @@ -122,10 +136,10 @@ func (g *predicateGroup) append(p Predicate) { } func (g *predicateGroup) setOperator(operatorT token, position int) ParseError { - if g.operator == nil { - g.operator = operatorT + if g.operator == -1 { + g.operator = operatorT.kind } - if g.operator != operatorT { + if g.operator != operatorT.kind { return parseError{ err: ErrCannotMixAndOr, position: position, @@ -142,9 +156,9 @@ func (g *predicateGroup) negateNextOperand() { func (g *predicateGroup) make() (Predicate, ParseError) { if len(g.ps) == 1 { return g.ps[0], nil - } else if g.operator == (tokenAnd{}) { + } else if g.operator == tokenAnd { return And{g.ps}, nil - } else if g.operator == (tokenOr{}) { + } else if g.operator == tokenOr { return Or{g.ps}, nil } else { // This would happen for an empty group.