Skip to content

Commit 5b4f773

Browse files
taokytaoky
authored
Extend grep functionality and apply grep filter for analyze (#26)
* Add URL contains, Time from & to filters * Apply grep filter to analyze package * Remove DateOnly in timeFormats
1 parent 29a50e8 commit 5b4f773

5 files changed

Lines changed: 112 additions & 52 deletions

File tree

‎pkg/analyze/analyze.go‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/schollz/progressbar/v3"
2020
"github.com/spf13/pflag"
2121
"github.com/taoky/ayano/pkg/fileiter"
22+
"github.com/taoky/ayano/pkg/grep"
2223
"github.com/taoky/ayano/pkg/parser"
2324
"github.com/taoky/ayano/pkg/util"
2425
)
@@ -153,16 +154,15 @@ type AnalyzerConfig struct {
153154
Parser string
154155
PrefixV4 int
155156
PrefixV6 int
156-
PrintDelta SizeFlag
157+
PrintDelta util.SizeFlag
157158
RefreshSec int
158159
RepeatWarn time.Duration
159-
Server string
160160
SortBy SortByFlag
161-
Threshold SizeFlag
162161
TopN int
163162
Truncate bool
164163
Truncate2 int
165164
Whole bool
165+
Filter grep.Filter
166166

167167
Analyze bool
168168
Daemon bool
@@ -181,13 +181,13 @@ func (c *AnalyzerConfig) InstallFlags(flags *pflag.FlagSet, cmdname string) {
181181
flags.IntVar(&c.PrefixV6, "prefixv6", c.PrefixV6, "Group IPv6 by prefix")
182182
flags.DurationVar(&c.RepeatWarn, "repeat-warn", c.RepeatWarn, "Highlight repeated URL visits longer than duration")
183183
flags.IntVarP(&c.RefreshSec, "refresh", "r", c.RefreshSec, "Refresh interval in seconds")
184-
flags.StringVarP(&c.Server, "server", "s", c.Server, "Server IP to filter (nginx-json only)")
185184
flags.VarP(&c.SortBy, "sort-by", "S", "Sort result by (size|requests)")
186-
flags.VarP(&c.Threshold, "threshold", "t", "Threshold size for request (only requests at least this large will be counted)")
187185
flags.IntVarP(&c.TopN, "top", "n", c.TopN, "Number of top items to show")
188186
flags.BoolVar(&c.Truncate, "truncate", c.Truncate, "Truncate long URLs from output")
189187
flags.IntVar(&c.Truncate2, "truncate-to", c.Truncate2, "Truncate URLs to given length, overrides --truncate")
190188

189+
c.Filter.InstallFlags(flags)
190+
191191
flags.StringVar(&c.CpuProfile, "cpuprof", c.CpuProfile, "Write CPU profiling information")
192192
flags.StringVar(&c.MemProfile, "memprof", c.MemProfile, "Write memory profiling information")
193193

@@ -209,14 +209,16 @@ func (c *AnalyzerConfig) UseLock() bool {
209209
}
210210

211211
func DefaultConfig() AnalyzerConfig {
212+
filter := grep.Filter{}
213+
filter.Threshold = util.SizeFlag(10e6)
212214
return AnalyzerConfig{
213215
Parser: "nginx-json",
214216
PrefixV4: 24,
215217
PrefixV6: 48,
216-
PrintDelta: SizeFlag(1e9),
218+
PrintDelta: util.SizeFlag(1e9),
217219
RefreshSec: 5,
218220
SortBy: SortBySize,
219-
Threshold: SizeFlag(10e6),
221+
Filter: filter,
220222
TopN: 10,
221223
}
222224
}
@@ -357,14 +359,8 @@ func (a *Analyzer) handleLogItem(logItem parser.LogItem) error {
357359
return nil
358360
}
359361

360-
// Filter by server
361-
if a.Config.Server != "" && logItem.Server != a.Config.Server {
362-
return nil
363-
}
364-
365-
// Filter by sent size
366-
size := logItem.Size
367-
if size < uint64(a.Config.Threshold) {
362+
// Filter
363+
if err := a.Config.Filter.Match(logItem); err != nil {
368364
return nil
369365
}
370366

@@ -385,7 +381,7 @@ func (a *Analyzer) handleLogItem(logItem parser.LogItem) error {
385381

386382
if a.Config.Analyze || a.Config.Daemon {
387383
// Avoid using double memory when not in interactive mode
388-
updateStats(StatKey{a.Config.Server, clientPrefix})
384+
updateStats(StatKey{a.Config.Filter.Server, clientPrefix})
389385
} else {
390386
updateStats(StatKey{logItem.Server, clientPrefix})
391387

@@ -396,7 +392,7 @@ func (a *Analyzer) handleLogItem(logItem parser.LogItem) error {
396392
}
397393

398394
if a.Config.Daemon {
399-
ipStats := a.stats[StatKey{a.Config.Server, clientPrefix}]
395+
ipStats := a.stats[StatKey{a.Config.Filter.Server, clientPrefix}]
400396
delta := ipStats.Size - ipStats.LastSize
401397
if ipStats.LastSize == 0 {
402398
ipStats.FirstSeen = logItem.Time
@@ -410,8 +406,8 @@ func (a *Analyzer) handleLogItem(logItem parser.LogItem) error {
410406
logItem.URL)
411407
}
412408
ipStats.LastSize += printTimes * uint64(a.Config.PrintDelta)
413-
// Just update [StatKey{a.Config.Server, clientPrefix}] here, as the config would not be updated runtime now
414-
a.stats[StatKey{a.Config.Server, clientPrefix}] = ipStats
409+
// Just update [StatKey{a.Config.Filter.Server, clientPrefix}] here, as the config would not be updated runtime now
410+
a.stats[StatKey{a.Config.Filter.Server, clientPrefix}] = ipStats
415411
}
416412

417413
if a.Config.DirAnalyze {

‎pkg/analyze/analyze_test.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ package analyze
33
import (
44
"os"
55
"testing"
6+
7+
"github.com/taoky/ayano/pkg/grep"
8+
"github.com/taoky/ayano/pkg/util"
69
)
710

811
func benchmarkAnalyzeLoop(b *testing.B, parserStr string) {
@@ -11,12 +14,13 @@ func benchmarkAnalyzeLoop(b *testing.B, parserStr string) {
1114
if logPath == "" {
1215
b.Fatal("LOG_PATH is not set")
1316
}
17+
filter := grep.Filter{}
18+
filter.Threshold = util.SizeFlag(100 * (1 << 20))
1419
c := AnalyzerConfig{
1520
NoNetstat: true,
1621
Parser: parserStr,
17-
Server: "",
1822
RefreshSec: 5,
19-
Threshold: 100 * (1 << 20),
23+
Filter: filter,
2024
TopN: 20,
2125
Whole: true,
2226

‎pkg/analyze/util.go‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,39 +6,10 @@ import (
66
"net/netip"
77
"path/filepath"
88
"slices"
9-
"strconv"
109
"strings"
1110
"time"
12-
13-
"github.com/dustin/go-humanize"
1411
)
1512

16-
type SizeFlag uint64
17-
18-
func (s SizeFlag) String() string {
19-
return humanize.Bytes(uint64(s))
20-
}
21-
22-
func (s *SizeFlag) Set(value string) error {
23-
// First try parsing as a plain number
24-
size, err := strconv.ParseUint(value, 10, 64)
25-
if err == nil {
26-
*s = SizeFlag(size)
27-
return nil
28-
}
29-
30-
size, err = humanize.ParseBytes(value)
31-
if err != nil {
32-
return err
33-
}
34-
*s = SizeFlag(size)
35-
return nil
36-
}
37-
38-
func (s SizeFlag) Type() string {
39-
return "size"
40-
}
41-
4213
type SortByFlag string
4314

4415
const (

‎pkg/grep/filter.go‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,28 @@ import (
44
"errors"
55
"fmt"
66
"net/netip"
7+
"strings"
8+
"time"
79

810
"github.com/spf13/pflag"
911
"github.com/taoky/ayano/pkg/parser"
12+
"github.com/taoky/ayano/pkg/util"
1013
)
1114

1215
type Filter struct {
13-
Prefixes []netip.Prefix
16+
Prefixes []netip.Prefix
17+
UrlContains []string
18+
TimeFrom time.Time
19+
TimeTo time.Time
20+
Threshold util.SizeFlag
21+
Server string
22+
}
23+
24+
var timeFormats = []string{
25+
time.RFC3339Nano,
26+
time.RFC3339,
27+
"2006-01-02 15:04:05",
28+
"20060102150405",
1429
}
1530

1631
func (f *Filter) InstallFlags(flags *pflag.FlagSet) {
@@ -23,15 +38,24 @@ func (f *Filter) InstallFlags(flags *pflag.FlagSet) {
2338
f.Prefixes = append(f.Prefixes, p)
2439
return nil
2540
})
41+
flags.StringArrayVar(&f.UrlContains, "url-contains", f.UrlContains, "URL substring to filter (can be specified multiple times)")
42+
flags.TimeVar(&f.TimeFrom, "time-from", f.TimeFrom, timeFormats, "Start time to filter (inclusive). Default value (zero) means no limit")
43+
flags.TimeVar(&f.TimeTo, "time-to", f.TimeTo, timeFormats, "End time to filter (inclusive). Default value (zero) means no limit")
44+
flags.VarP(&f.Threshold, "threshold", "t", "Threshold size for request (only requests at least this large will be counted)")
45+
flags.StringVarP(&f.Server, "server", "s", f.Server, "Server IP to filter (nginx-json only)")
2646
}
2747

2848
func (f *Filter) IsEmpty() bool {
29-
return len(f.Prefixes) == 0
49+
return len(f.Prefixes) == 0 && len(f.UrlContains) == 0 && f.TimeFrom.IsZero() && f.TimeTo.IsZero() && f.Threshold == 0 && f.Server == ""
3050
}
3151

3252
var (
3353
ErrInvalidIP = errors.New("invalid client IP")
3454
ErrNoPrefixMatch = errors.New("no matching prefix")
55+
ErrURLNoMatch = errors.New("URL does not match")
56+
ErrTimeNoMatch = errors.New("time does not match")
57+
ErrSizeTooSmall = errors.New("size below threshold")
58+
ErrServerNoMatch = errors.New("server does not match")
3559
)
3660

3761
func (f *Filter) Match(item parser.LogItem) error {
@@ -51,5 +75,37 @@ func (f *Filter) Match(item parser.LogItem) error {
5175
return ErrNoPrefixMatch
5276
}
5377
}
78+
if len(f.UrlContains) > 0 {
79+
urlMatch := false
80+
for _, substr := range f.UrlContains {
81+
if strings.Contains(item.URL, substr) {
82+
urlMatch = true
83+
break
84+
}
85+
}
86+
if !urlMatch {
87+
return ErrURLNoMatch
88+
}
89+
}
90+
if !f.TimeFrom.IsZero() {
91+
if item.Time.Before(f.TimeFrom) {
92+
return ErrTimeNoMatch
93+
}
94+
}
95+
if !f.TimeTo.IsZero() {
96+
if item.Time.After(f.TimeTo) {
97+
return ErrTimeNoMatch
98+
}
99+
}
100+
if f.Threshold > 0 {
101+
if item.Size < uint64(f.Threshold) {
102+
return ErrSizeTooSmall
103+
}
104+
}
105+
if f.Server != "" {
106+
if item.Server != f.Server {
107+
return ErrServerNoMatch
108+
}
109+
}
54110
return nil
55111
}

‎pkg/util/flag.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package util
2+
3+
import (
4+
"strconv"
5+
6+
"github.com/dustin/go-humanize"
7+
)
8+
9+
type SizeFlag uint64
10+
11+
func (s SizeFlag) String() string {
12+
return humanize.Bytes(uint64(s))
13+
}
14+
15+
func (s *SizeFlag) Set(value string) error {
16+
// First try parsing as a plain number
17+
size, err := strconv.ParseUint(value, 10, 64)
18+
if err == nil {
19+
*s = SizeFlag(size)
20+
return nil
21+
}
22+
23+
size, err = humanize.ParseBytes(value)
24+
if err != nil {
25+
return err
26+
}
27+
*s = SizeFlag(size)
28+
return nil
29+
}
30+
31+
func (s SizeFlag) Type() string {
32+
return "size"
33+
}

0 commit comments

Comments
 (0)